def build_args(parser):
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--config", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint")
parser.add_argument("-j",
"--workers",
default=None,
type=int,
metavar="N",
help="number of data loading workers")
parser.add_argument("-b",
"--batch-size",
default=None,
type=int,
metavar="N",
help="mini-batch size")
boolean_flag(parser, "verbose", default=False)
parser.add_argument("--out-prefix", type=str, default=None)
return parser
def build_args(parser):
parser.add_argument("-C",
"--config",
help="path to config/configs",
required=True)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("-j",
"--num-workers",
default=None,
type=int,
help="number of data loading workers")
parser.add_argument("-b",
"--batch-size",
default=None,
type=int,
help="mini-batch size")
parser.add_argument("-e",
"--num-epochs",
default=None,
type=int,
help="number of epochs")
parser.add_argument("--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint")
parser.add_argument("--seed", type=int, default=42)
boolean_flag(parser, "verbose", default=False)
boolean_flag(parser, "check", default=False)
return parser
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--model-dir", type=str, default=None)
parser.add_argument("--config", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint")
parser.add_argument("-j",
"--workers",
default=None,
type=int,
metavar="N",
help="number of data loading workers")
parser.add_argument("-b",
"--batch-size",
default=None,
type=int,
metavar="N",
help="mini-batch size")
boolean_flag(parser, "verbose", default=False)
parser.add_argument("--out-prefix", type=str, default=None)
args, unknown_args = parser.parse_known_args()
return args, unknown_args
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--in-csv", type=str)
parser.add_argument("--fasttext-model", type=str, default=None)
parser.add_argument("--w2v-model", type=str, default=None)
boolean_flag(parser, "normalize", default=False)
parser.add_argument("--txt-sep", type=str, default=" ")
parser.add_argument("--txt-col", type=str)
parser.add_argument("--out-npy", type=str, dest="out_npy", required=True)
parser.add_argument("--n-workers",
type=int,
dest="n_workers",
help="count of workers for dataloader",
default=4)
parser.add_argument("--batch-size",
type=int,
dest="batch_size",
help="dataloader batch size",
default=128)
parser.add_argument("--verbose",
dest="verbose",
action="store_true",
default=False)
args = parser.parse_args()
return args
def build_args(parser):
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--resume", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument(
"--vis",
type=int,
default=None)
parser.add_argument(
"--infer",
type=int,
default=None)
parser.add_argument(
"--train",
type=int,
default=None)
boolean_flag(parser, "check", default=False)
boolean_flag(parser, "db", default=True)
return parser
def add_arguments(parser) -> None:
"""Function to add common arguments to argparse:
feature_dim: Feature dim for latent vector
temperature: Temperature used in softmax
batch_size: Number of images in each mini-batch
epochs: Number of sweeps over the dataset to train
num_workers: Number of workers to process a dataloader
logdir: Logs directory (tensorboard, weights, etc)
dataset: CIFAR-10, CIFAR-100 or STL10
learning-rate: Learning rate for optimizer
Args:
parser: argparser like object
"""
parser.add_argument(
"--dataset",
default="CIFAR-10",
type=str,
choices=DATASETS.keys(),
help="Dataset: CIFAR-10, CIFAR-100 or STL10",
)
parser.add_argument(
"--logdir",
default="./logdir",
type=str,
help="Logs directory (tensorboard, weights, etc)",
)
parser.add_argument(
"--epochs",
default=1000,
type=int,
help="Number of sweeps over the dataset to train",
)
parser.add_argument(
"--num-workers",
default=1,
type=float,
help="Number of workers to process a dataloader",
)
parser.add_argument("--batch-size",
default=512,
type=int,
help="Number of images in each mini-batch")
parser.add_argument("--feature-dim",
default=128,
type=int,
help="Feature dim for latent vector")
parser.add_argument("--temperature",
default=0.5,
type=float,
help="Temperature used in softmax")
parser.add_argument("--learning-rate",
default=0.001,
type=float,
help="Learning rate for optimizer")
# boolean_flag(parser=parser, name="check", default=False)
boolean_flag(parser=parser, name="verbose", default=False)
def build_args(parser):
"""
Constructs the command-line arguments for ``catalyst-contrib tag2label``.
Args:
parser: current parser
Returns:
updated parser
"""
parser.add_argument("--in-csv",
type=str,
default=None,
help="Path to data in `.csv`.")
parser.add_argument(
"--in-dir",
type=str,
default=None,
help="Path to directory with dataset" +
"or paths separated by commas for several datasets",
)
parser.add_argument(
"--out-dataset",
type=str,
default=None,
required=True,
help="Path to output dataframe",
)
parser.add_argument(
"--out-labeling",
type=str,
default=None,
required=True,
help="Path to output JSON",
)
parser.add_argument("--tag-column",
type=str,
default="tag",
help="Target column name")
parser.add_argument(
"--tag-delim",
type=str,
default=None,
help="Separator if you want to use several target columns",
)
boolean_flag(
parser,
"recursive",
default=False,
help="Include subdirs in dataset",
)
return parser
def build_args(parser):
parser.add_argument("--config", type=str, required=True)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--algorithm", type=str, default=None)
parser.add_argument("--environment", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--resume", type=str, default=None)
parser.add_argument(
"--vis",
type=int,
default=None)
parser.add_argument(
"--infer",
type=int,
default=None)
parser.add_argument(
"--train",
type=int,
default=None)
parser.add_argument(
"--action-noise-prob",
type=float,
default=None)
parser.add_argument(
"--param-noise-prob",
type=float,
default=None)
parser.add_argument(
"--max-noise-power",
type=float,
default=None)
parser.add_argument(
"--max-action-noise",
type=float,
default=None)
parser.add_argument(
"--max-param-noise",
type=float,
default=None)
boolean_flag(parser, "debug", default=False)
boolean_flag(parser, "redis", default=True)
return parser
def build_args(parser):
"""
Constructs the command-line arguments for
``catalyst-contrib process-images``.
Args:
parser: current parser
Returns:
updated parser
"""
parser.add_argument("--in-dir",
required=True,
type=Path,
help="Raw data folder path")
parser.add_argument(
"--out-dir",
required=True,
type=Path,
help="Processed images folder path",
)
parser.add_argument(
"--num-workers",
"-j",
default=1,
type=int,
help="Number of workers to parallel the processing",
)
parser.add_argument(
"--max-size",
default=None,
required=False,
type=int,
help="Output images size. E.g. 224, 448",
)
boolean_flag(parser, "clear-exif", default=True, help="Clear EXIF data")
boolean_flag(parser,
"grayscale",
default=False,
help="Read images in grayscale")
boolean_flag(
parser,
"expand-dims",
default=True,
help="Expand array shape for grayscale images",
)
return parser
def build_args(parser: ArgumentParser):
"""Constructs the command-line arguments for ``catalyst-dl run``."""
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True,
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--baselogdir", type=str, default=None)
parser.add_argument(
"-j",
"--num-workers",
default=None,
type=int,
help="number of data loading workers",
)
parser.add_argument("-b",
"--batch-size",
default=None,
type=int,
help="mini-batch size")
parser.add_argument("-e",
"--num-epochs",
default=None,
type=int,
help="number of epochs")
parser.add_argument(
"--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint",
)
# parser.add_argument(
# "--autoresume",
# type=str,
# help=(
# "try automatically resume from logdir//{best,last}_full.pth "
# "if --resume is empty"
# ),
# required=False,
# choices=["best", "last"],
# default=None,
# )
parser.add_argument("--seed", type=int, default=42)
boolean_flag(
parser,
"apex",
default=os.getenv("USE_APEX", "0") == "1",
help="Enable/disable using of Apex extension",
)
boolean_flag(
parser,
"amp",
default=os.getenv("USE_AMP", "0") == "1",
help="Enable/disable using of PyTorch AMP extension",
)
# boolean_flag(
# parser,
# "distributed",
# shorthand="ddp",
# default=os.getenv("USE_DDP", "0") == "1",
# help="Run in distributed mode",
# )
boolean_flag(parser, "verbose", default=None)
boolean_flag(parser, "timeit", default=None)
# boolean_flag(parser, "check", default=None)
# boolean_flag(parser, "overfit", default=None)
boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
boolean_flag(parser, "benchmark", default=None, help="Use CuDNN benchmark")
parser.add_argument("--storage", type=int, default=None)
parser.add_argument("--study-name", type=int, default=None)
parser.add_argument("--n-trials", type=int, default=None)
parser.add_argument("--timeout", type=int, default=None)
parser.add_argument("--n-jobs", type=int, default=None)
boolean_flag(parser, "gc-after-trial", default=False)
boolean_flag(parser, "show-progress-bar", default=False)
return parser
def build_args(parser):
"""Constructs the command-line arguments."""
parser.add_argument(
"--in-csv",
type=Path,
help="Path to .csv with labels column",
required=True,
)
parser.add_argument(
"--in-label-column",
type=str,
help="Column to get labels",
required=False,
default="labels",
)
parser.add_argument(
"--in-npy",
type=Path,
help="Path to .npy with class logits",
required=True,
)
parser.add_argument(
"--out-thresholds",
type=Path,
help="Path to save .json with thresholds",
required=True,
)
parser.add_argument(
"--metric",
type=str,
help="Metric to use",
required=False,
choices=_BINARY_PER_CLASS_METRICS,
default="roc_auc_score",
)
# parser.add_argument(
# "--ignore-label", type=int,
# required=False,
# default=None
# )
parser.add_argument("--num-splits",
type=int,
help="NUM_SPLITS",
required=False,
default=5)
parser.add_argument(
"--num-repeats",
type=int,
help="NUM_REPEATS",
required=False,
default=1,
)
parser.add_argument(
"--num-workers",
type=int,
help="CPU pool size",
required=False,
default=1,
)
boolean_flag(parser, "verbose", default=False)
boolean_flag(parser, "sigmoid", default=False)
return parser
def build_args(parser: ArgumentParser):
"""Constructs the command-line arguments for ``catalyst-dl run``."""
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=False,
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--baselogdir", type=str, default=None)
# parser.add_argument(
# "--resume", default=None, type=str, metavar="PATH", help="path to latest checkpoint",
# )
# parser.add_argument(
# "--autoresume",
# type=str,
# help=(
# "try automatically resume from logdir//{best,last}_full.pth " "if --resume is empty"
# ),
# required=False,
# choices=["best", "last"],
# default=None,
# )
parser.add_argument("--seed", type=int, default=42)
boolean_flag(
parser,
"apex",
default=os.getenv("USE_APEX", "0") == "1",
help="Enable/disable using of Nvidia Apex extension",
)
boolean_flag(
parser,
"amp",
default=os.getenv("USE_AMP", "0") == "1",
help="Enable/disable using of PyTorch AMP extension",
)
boolean_flag(
parser,
"fp16",
default=os.getenv("USE_FP16", "0") == "1",
help="Run in half-precision mode",
)
boolean_flag(
parser, "ddp", default=os.getenv("USE_DDP", "0") == "1", help="Run in distributed mode"
)
boolean_flag(parser, "verbose", default=None)
boolean_flag(parser, "timeit", default=None)
boolean_flag(parser, "check", default=None)
boolean_flag(parser, "overfit", default=None)
boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
boolean_flag(parser, "benchmark", default=None, help="Use CuDNN benchmark")
boolean_flag(parser, "hydra", default=None, help="Use Hydra")
return parser
"--param-noise-prob",
type=float,
default=None)
parser.add_argument(
"--max-noise-power",
type=float,
default=None)
parser.add_argument(
"--max-action-noise",
type=float,
default=None)
parser.add_argument(
"--max-param-noise",
type=float,
default=None)
boolean_flag(parser, "debug", default=False)
args = parser.parse_args()
args, config = parse_args_uargs(args, [])
env_module = import_module("env_module", args.environment)
algo_module = import_module("algo_module", args.algorithm)
def run_sampler(
*,
config, vis, infer,
action_noise_prob, param_noise_prob,
action_noise=None, param_noise=None,
noise_power=None, # @TODO: remove
id=None, resume=None, debug=False):
def build_args(parser):
"""
Constructs the command-line arguments for
``catalyst-contrib image2embeddings``.
Args:
parser: parser
Returns:
modified parser
"""
parser.add_argument("--in-csv",
type=str,
dest="in_csv",
help="Path to csv with photos")
parser.add_argument(
"--img-rootpath",
type=str,
dest="rootpath",
help="Path to photos directory",
)
parser.add_argument(
"--img-col",
type=str,
dest="img_col",
help="Column in table that contain image path",
)
parser.add_argument(
"--img-size",
type=int,
dest="img_size",
default=224,
help="Target size of images",
)
parser.add_argument(
"--out-npy",
type=str,
dest="out_npy",
required=True,
help="Path to output `.npy` file with embedded features",
)
parser.add_argument(
"--arch",
type=str,
dest="arch",
default="resnet18",
help="Neural network architecture",
)
parser.add_argument(
"--pooling",
type=str,
dest="pooling",
default="GlobalAvgPool2d",
help="Type of pooling to use",
)
parser.add_argument(
"--traced-model",
type=Path,
dest="traced_model",
default=None,
help="Path to pytorch traced model",
)
parser.add_argument(
"--num-workers",
type=int,
dest="num_workers",
help="Count of workers for dataloader",
default=0,
)
parser.add_argument(
"--batch-size",
type=int,
dest="batch_size",
help="Dataloader batch size",
default=32,
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Print additional information",
)
parser.add_argument("--seed", type=int, default=42)
boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
boolean_flag(parser, "benchmark", default=None, help="Use CuDNN benchmark")
return parser
def build_args(parser):
"""
Constructs the command-line arguments for
``catalyst-contrib text2embeddings``.
Args:
parser: parser
Returns:
modified parser
"""
parser.add_argument("--in-csv",
type=str,
help="Path to csv with text",
required=True)
parser.add_argument("--txt-col",
type=str,
help="Column in table that contain text")
parser.add_argument(
"--in-huggingface",
type=str,
required=False,
help="model from huggingface hub",
)
required_path_to_model = True
for arg in sys.argv:
if "--in-huggingface" in arg:
required_path_to_model = False
if required_path_to_model:
parser.add_argument("--in-config",
type=Path,
required=required_path_to_model)
parser.add_argument("--in-model",
type=Path,
required=required_path_to_model)
parser.add_argument("--in-vocab",
type=Path,
required=required_path_to_model)
parser.add_argument(
"--out-prefix",
type=str,
required=True,
)
parser.add_argument("--max-length", type=int, default=512) # noqa: WPS432
boolean_flag(parser, "mask-for-max-length", default=False)
boolean_flag(parser, "output-hidden-states", default=False)
parser.add_argument(
"--bert-level",
type=int,
help="BERT features level to use",
default=None, # noqa: WPS432
)
boolean_flag(parser, "strip", default=True)
boolean_flag(parser, "lowercase", default=True)
boolean_flag(parser, "remove-punctuation", default=True)
parser.add_argument("--pooling", type=str, default="avg")
parser.add_argument(
"--num-workers",
type=int,
dest="num_workers",
help="Count of workers for dataloader",
default=0,
)
parser.add_argument(
"--batch-size",
type=int,
dest="batch_size",
help="Dataloader batch size",
default=32, # noqa: WPS432
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Print additional information",
)
parser.add_argument("--seed", type=int, default=42) # noqa: WPS432
boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
boolean_flag(parser, "benchmark", default=None, help="Use CuDNN benchmark")
boolean_flag(
parser,
"force-save",
default=None,
help="Force save `.npy` with np.save",
)
return parser
