def arg_setting(): parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=False) group.add_argument('--vae_path', type=str, help='path to your trained discrete VAE') group.add_argument('--dalle_path', type=str, help='path to your partially trained DALL-E') parser.add_argument( '--image_text_folder', type=str, default='/opt/ml/input/data/training', help='path to your folder of images and text for learning the DALL-E') parser.add_argument( '--truncate_captions', dest='truncate_captions', action='store_true', help= 'Captions passed in which exceed the max token length will be truncated if this is set.' ) parser.add_argument('--random_resize_crop_lower_ratio', dest='resize_ratio', type=float, default=0.75, help='Random resized crop lower ratio') parser.add_argument('--chinese', dest='chinese', action='store_true') parser.add_argument('--taming', dest='taming', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False # action='store_true' ) parser.add_argument('--hug', dest='hug', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False) parser.add_argument('--bpe_path', type=str, help='path to your huggingface BPE json file') parser.add_argument( '--fp16', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False, # action='store_true', help='(experimental) - Enable DeepSpeed 16 bit precision. Reduces VRAM.') parser.add_argument( '--wandb_name', default='dalle_train_transformer', help= 'Name W&B will use when saving results.\ne.g. `--wandb_name "coco2017-full-sparse"`' ) parser = distributed_utils.wrap_arg_parser(parser) train_group = parser.add_argument_group('Training settings') train_group.add_argument('--epochs', default=20, type=int, help='Number of epochs') train_group.add_argument('--batch_size', default=4, type=int, help='Batch size') train_group.add_argument('--learning_rate', default=3e-4, type=float, help='Learning rate') train_group.add_argument('--clip_grad_norm', default=0.5, type=float, help='Clip gradient norm') train_group.add_argument('--lr_decay', dest='lr_decay', action='store_true') model_group = parser.add_argument_group('Model settings') model_group.add_argument('--dim', default = 512, type = int, help = 'Model dimension') model_group.add_argument('--heads', default=8, type=int, help='Model number of heads') model_group.add_argument('--dim_head', default=64, type=int, help='Model head dimension') model_group.add_argument('--reversible', dest='reversible', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False # action='store_true' ) model_group.add_argument('--loss_img_weight', default=7, type=int, help='Image loss weight') model_group.add_argument('--text_seq_len', default=256, type=int, help='Text sequence length') model_group.add_argument('--depth', default=2, type=int, help='Model depth') model_group.add_argument( '--attn_types', default='full', type=str, help= 'comma separated list of attention types. attention type can be: full or sparse or axial_row or axial_col or conv_like.' ) parser.add_argument('--num_worker', type=int, default=4) parser.add_argument('--model_dir', type=str, default='model/dalle/') parser.add_argument('--num-gpus', type=int, default=8) parser.add_argument('--hosts', type=list, default=[]) parser.add_argument('--current-host', type=str, default="") parser.add_argument( '--backend', type=str, default='nccl', help= 'backend for distributed training (tcp, gloo on cpu and gloo, nccl on gpu)' ) # Setting for Model Parallel parser.add_argument("--sagemakermp", type=lambda s:s.lower() in ['true','t','yes','1'], default=False) parser.add_argument("--num_microbatches", type=int, default=4) parser.add_argument("--num_partitions", type=int, default=2) parser.add_argument("--horovod", type=bool, default=False) parser.add_argument("--ddp", type=bool, default=True) parser.add_argument("--amp", type=int, default=0) ## if amp is 1, true parser.add_argument("--pipeline", type=str, default="interleaved") parser.add_argument("--optimize", type=str, default="speed") parser.add_argument("--placement_strategy", type=str, default="spread") parser.add_argument("--assert-losses", type=bool, default=False) parser.add_argument('--mp_parameters', type=str, default='') parser.add_argument("--partial-checkpoint", type=str, default="", help="The checkpoint path to load") parser.add_argument("--full-checkpoint", type=str, default="", help="The checkpoint path to load") parser.add_argument("--save-full-model", action="store_true", default=False, help="For Saving the current Model") parser.add_argument( "--save-partial-model", action="store_true", default=False, help="For Saving the current Model", ) args = parser.parse_args() return args
dest='resize_ratio', type=float, default=0.75, help='Random resized crop lower ratio') parser.add_argument('--chinese', dest='chinese', action='store_true') parser.add_argument('--taming', dest='taming', action='store_true') parser.add_argument('--bpe_path', type=str, help='path to your huggingface BPE json file') parser.add_argument('--fp16', action='store_true') parser = distributed_utils.wrap_arg_parser(parser) args = parser.parse_args() # helpers def exists(val): return val is not None # constants VAE_PATH = args.vae_path DALLE_PATH = args.dalle_path RESUME = exists(DALLE_PATH)
def arg_setting(): parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group(required=False) group.add_argument('--vae_path', type=str, help='path to your trained discrete VAE') group.add_argument('--dalle_path', type=str, help='path to your partially trained DALL-E') parser.add_argument( '--image_text_folder', type=str, default='../../CUB_BIRD', help='path to your folder of images and text for learning the DALL-E') parser.add_argument( '--truncate_captions', dest='truncate_captions', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False, help= 'Captions passed in which exceed the max token length will be truncated if this is set.' ) # parser.add_argument( # '--truncate_captions', # dest='truncate_captions', # action='store_true', # help= # 'Captions passed in which exceed the max token length will be truncated if this is set.' # ) parser.add_argument('--random_resize_crop_lower_ratio', dest='resize_ratio', type=float, default=0.75, help='Random resized crop lower ratio') parser.add_argument('--chinese', dest='chinese', action='store_true') parser.add_argument( '--taming', dest='taming', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], # action='store_true' ) parser.add_argument('--hug', dest='hug', action='store_true') parser.add_argument('--bpe_path', type=str, help='path to your huggingface BPE json file') parser.add_argument('--dalle_output_file_name', type=str, default="dalle.pt", help='output_file_name') parser.add_argument( '--fp16', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False, # action='store_true', help='(experimental) - Enable DeepSpeed 16 bit precision. Reduces VRAM.' ) parser.add_argument( '--wandb_name', default='dalle_train_transformer', help= 'Name W&B will use when saving results.\ne.g. `--wandb_name "coco2017-full-sparse"`' ) parser = distributed_utils.wrap_arg_parser(parser) train_group = parser.add_argument_group('Training settings') train_group.add_argument('--epochs', default=20, type=int, help='Number of epochs') train_group.add_argument('--save_every_n_steps', default=1000, type=int, help='Save a checkpoint every n steps') train_group.add_argument('--batch_size', default=4, type=int, help='Batch size') train_group.add_argument('--learning_rate', default=3e-4, type=float, help='Learning rate') train_group.add_argument('--clip_grad_norm', default=0.5, type=float, help='Clip gradient norm') train_group.add_argument('--lr_decay', dest='lr_decay', action='store_true') model_group = parser.add_argument_group('Model settings') model_group.add_argument('--dim', default=512, type=int, help='Model dimension') model_group.add_argument('--heads', default=8, type=int, help='Model number of heads') model_group.add_argument('--dim_head', default=64, type=int, help='Model head dimension') model_group.add_argument( '--reversible', dest='reversible', type=lambda s: s.lower() in ['true', 't', 'yes', '1'], default=False # action='store_true' ) model_group.add_argument('--loss_img_weight', default=7, type=int, help='Image loss weight') model_group.add_argument('--text_seq_len', default=256, type=int, help='Text sequence length') model_group.add_argument('--depth', default=2, type=int, help='Model depth') model_group.add_argument( '--attn_types', default='full', type=str, help= 'comma separated list of attention types. attention type can be: full or sparse or axial_row or axial_col or conv_like.' ) parser.add_argument('--num_worker', type=int, default=4) parser.add_argument('--model_dir', type=str, default='model/dalle/') parser.add_argument('--num-gpus', type=int, default=8) parser.add_argument('--hosts', type=list, default=[]) parser.add_argument('--current-host', type=str, default="") parser.add_argument( '--backend', type=str, default='nccl', help= 'backend for distributed training (tcp, gloo on cpu and gloo, nccl on gpu)' ) parser.add_argument('--output_s3', type=str, default="s3://") args = parser.parse_args() return args