def configure(self, parser: ArgumentParser): # parser.add_argument( # '--name', default=self.image.name, type=str # ) parser.add_argument('--profile', default='default', type=str) parser.add_argument('--path', default=self.image.path, type=str) parser.add_argument('--tag', default=self.image.tag, type=str) parser.add_argument('--accounts', default=",".join(self.image.accounts), type=str) bool_argument(parser, '--force', default=True, help="If image already exists in ECR, do nothing") bool_argument(parser, '--pull', default=True, help="Pull latest version of FROM images if building") bool_argument(parser, '--push', default=True, help="Push results to ECR") bool_argument(parser, '--cache', default=True, help='Use cached layers during build') bool_argument(parser, '--wsl', default=False, help='WSL path fix') for k, v in self.image.download_files.items(): flag = "--{}".format(k.replace("_", "-")) parser.add_argument(flag, default=v, type=str, help=f"Download auxiliary file (default: {v})") return super().configure(parser)
def train_args(parser, max_epochs=10, n_saved=10, save_event='EPOCH_COMPLETED'): parser.add_argument( '--max-epochs', type=int, default=max_epochs, metavar='N', help='number of epochs to train (default: {})'.format(max_epochs)) parser.add_argument( '--n-saved', default=n_saved, type=int, help='Number of checkpoints to keep (default: ``{}``)'.format(n_saved)) parser.add_argument('--save-event', default=save_event, help='save event') bool_argument(parser, '--train-pbar', default=True, help='Enable train progress bar') parser.add_argument('--train-print-event', default=None, help='training print event') parser.add_argument('--train-log-event', default=None, help='training log event')
def mlflow_args(parser, mlflow_enable=True, mlflow_tracking_uri=None, mlflow_tracking_password=None, mlflow_tracking_username=None, mlflow_tracking_secret_name=None, mlflow_tracking_secret_region=None, mlflow_tracking_secret_profile=None, mlflow_experiment_name='default', mlflow_run_name=None): bool_argument( parser, '--mlflow-enable', default=mlflow_enable, help='Enable logging to MLflow (default: {})'.format(mlflow_enable)) parser.add_argument('--mlflow-experiment-name', default=mlflow_experiment_name, help='Experiment name in MLflow (default: {})'.format( mlflow_experiment_name)) parser.add_argument( '--mlflow-run-name', default=mlflow_run_name, help='Run name in MLflow (default: {})'.format(mlflow_run_name)) parser.add_argument( '--mlflow-tracking-uri', default=mlflow_tracking_uri, help='URI of MLflow tracking server (default: ``{}``)'.format( mlflow_tracking_uri)) parser.add_argument( '--mlflow-tracking-username', default=mlflow_tracking_username, type=str, help='Username for MLflow tracking server (default: ``{}``)'.format( mlflow_tracking_username)) parser.add_argument( '--mlflow-tracking-password', default=mlflow_tracking_password, type=str, help='Password for MLflow tracking server (default: ``{}``)'.format( mlflow_tracking_password)) parser.add_argument( '--mlflow-tracking-secret-name', default=mlflow_tracking_secret_name, type=str, help='Secret for accessing MLflow (default: ``{}``)'.format( mlflow_tracking_secret_name)) parser.add_argument( '--mlflow-tracking-secret-profile', default=mlflow_tracking_secret_profile, type=str, help= 'Profile for accessing secret for accessing MLflow (default: ``{}``)'. format(mlflow_tracking_secret_profile)) parser.add_argument( '--mlflow-tracking-secret-region', default=mlflow_tracking_secret_region, type=str, help='Region for accessing secret for accessing MLflow (default: ``{}``)' .format(mlflow_tracking_secret_region))
def train_args(parser: argparse.ArgumentParser, batch_size=64, eval_batch_size=64, max_epochs=100, learning_rate=0.0003, device=None, mlflow_enable=True, mlflow_tracking_uri=None, mlflow_experiment_name='default', mlflow_run_name=None, arguments='all', n_saved=5): r""" Configure arguments for training Parameters ---------- parser : argparse.ArgumentParser Argument parser to configure. batch_size : int Batch size. arguments : list List of arguments to include (e.g. ``['batch_size``]) or the string ``"all"`` to include all arguments. (default: "all") sagemaker_args : dict Keyword arguments to ``aws-sagemaker-remote.training.main.sagemaker_training_main``. See `aws_sagemaker_remote <https://aws-sagemaker-remote.readthedocs.io/en/latest/>`_. """ if arguments == 'all' or 'workers' in arguments: parser.add_argument("--workers", type=int, default=2, help="number of data loading workers (default: 2)") if arguments == 'all' or 'batch_size' in arguments: parser.add_argument( '--batch-size', type=int, default=batch_size, metavar='N', help='input batch size for training (default: {})'.format( batch_size)) if arguments == 'all' or 'test_batch_size' in arguments: parser.add_argument( '--test-batch-size', type=int, default=eval_batch_size, metavar='N', help='input batch size for testing (default: {})'.format( eval_batch_size)) if arguments == 'all' or 'max_epochs' in arguments: parser.add_argument( '--max-epochs', type=int, default=max_epochs, metavar='N', help='number of epochs to train (default: {})'.format(max_epochs)) if arguments == 'all' or 'learning_rate' in arguments: parser.add_argument( '--learning-rate', type=float, default=learning_rate, metavar='LR', help='learning rate (default: {})'.format(learning_rate)) if arguments == 'all' or 'seed' in arguments: parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') if arguments == 'all' or 'mlflow_enable' in arguments: bool_argument(parser, '--mlflow-enable', default=mlflow_enable, help='Enable logging to MLflow (default: {})'.format( mlflow_enable)) if arguments == 'all' or 'mlflow_experiment_name' in arguments: parser.add_argument( '--mlflow-experiment-name', default=mlflow_experiment_name, help='Experiment name in MLflow (default: {})'.format( mlflow_experiment_name)) if arguments == 'all' or 'mlflow_run_name' in arguments: parser.add_argument( '--mlflow-run-name', default=mlflow_run_name, help='Run name in MLflow (default: {})'.format(mlflow_run_name)) if arguments == 'all' or 'mlflow_tracking_uri' in arguments: parser.add_argument( '--mlflow-tracking-uri', default=mlflow_tracking_uri, help='URI of MLflow tracking server (default: ``{}``)'.format( mlflow_tracking_uri)) if arguments == 'all' or 'n_saved' in arguments: parser.add_argument( '--n-saved', default=n_saved, type=int, help='Number of checkpoints to keep (default: ``{}``)'.format( n_saved)) if arguments == 'all' or 'device' in arguments: if device is None: device = "cpu" if not torch.cuda.is_available() else "cuda" parser.add_argument("--device", type=str, default=device, help="device to use (default: {})".format(device)) """
def batch_argparse_callback(parser: argparse.ArgumentParser, config: BatchConfig): parser.add_argument('--profile', type=str, default=config.profile, help='AWS profile name') parser.add_argument('--output-json', type=str, default=None, help='Output job information to JSON file') parser.add_argument( '--stack-name', type=str, default=config.stack_name, help= f'AWS CloudFormation stack name to which resources are deployed (default: {config.stack_name})', required=not config.stack_name) parser.add_argument( '--code-dir', type=str, default=config.code_dir, help=f'Directory of Lambda code (default: {config.code_dir})', required=not config.code_dir) bool_argument( parser, '--deploy', default=False, help='Force Lambda deployment even if function already exists') bool_argument( parser, '--deploy-only', default=False, help= 'Deploy and exit. Use `--deploy yes --deploy-only yes` to force deployment and exit' ) bool_argument(parser, '--confirmation-required', default=True, help='Require confirmation in console to run job') bool_argument(parser, '--development', default=config.development, help='Webpack in development mode') parser.add_argument( '--manifest', type=str, default=config.manifest, help= 'File manifest to process. Must be a CSV with first column containing an S3 bucket and second column containing an S3 key.', required=not config.manifest) parser.add_argument('--report', type=str, default=config.report, help='S3 path to store report') parser.add_argument('--description', type=str, default=config.description, help='Description of batch job') parser.add_argument('--timeout', type=int, default=config.timeout, help='Hard timeout of Lambda in seconds') parser.add_argument( '--ignore', type=int, default=0, help= 'Number of columns of input CSV to ignore. Job will fail if CSV does not have 2+ignore columns. For example, if your CSV has bucket, key, and 5 more columns set ignore to 5.' ) parser.add_argument('--memory', type=int, default=128, help='Memory to allocate') if config.support_soft: parser.add_argument('--soft-timeout', type=int, default=config.soft_timeout, help='Soft timeout in seconds') # parser.add_argument( # '--role-name', type=str, default=config.role_name, help='S3 path to store report' # ) if config.argparse_callback: config.argparse_callback(parser)
def eval_args(parser: argparse.ArgumentParser): bool_argument(parser, '--eval-pbar', default=True, help='Enable eval progress bar')
def batch_argparse_callback(parser: argparse.ArgumentParser, config: BatchConfig): parser.add_argument('--profile', type=str, default=config.profile, help='AWS profile name') parser.add_argument('--output-json', type=str, default=None, help='Output job information to JSON file') parser.add_argument('--stack-name', type=str, default=config.stack_name, help='Stack name for deploying Lambda', required=not config.stack_name) parser.add_argument('--code-dir', type=str, default=config.code_dir, help='Directory of Lambda code', required=not config.code_dir) bool_argument( parser, '--deploy', default=False, help='Force Lambda deployment even if function already exists') bool_argument( parser, '--deploy-only', default=False, help= 'Deploy and exit. Use `--deploy yes --deploy-only yes` to force deployment and exit' ) bool_argument(parser, '--confirmation-required', default=True, help='Require confirmation in console to run job') bool_argument(parser, '--development', default=config.development, help='Require confirmation in console to run job') parser.add_argument('--manifest', type=str, default=config.manifest, help='File manifest to process', required=not config.manifest) parser.add_argument('--report', type=str, default=config.report, help='S3 path to store report') parser.add_argument('--description', type=str, default=config.description, help='S3 path to store report') parser.add_argument('--timeout', type=int, default=config.timeout, help='S3 path to store report') parser.add_argument('--ignore', type=int, default=0, help='Columns to ignore') parser.add_argument('--memory', type=int, default=128, help='Memory to allocate') parser.add_argument('--soft-timeout', type=int, default=config.soft_timeout, help='S3 path to store report') # parser.add_argument( # '--role-name', type=str, default=config.role_name, help='S3 path to store report' # ) if config.argparse_callback: config.argparse_callback(parser)