def build_args(parser):
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
# parser.add_argument("--resume", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--train", type=int, default=None)
parser.add_argument("--valid", type=int, default=None)
parser.add_argument("--infer", type=int, default=None)
parser.add_argument("--vis", type=int, default=None)
boolean_flag(parser, "check", default=False)
boolean_flag(parser, "db", default=True)
return parser
def build_args(parser):
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--resume", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
boolean_flag(
parser, "deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend"
)
boolean_flag(
parser, "benchmark",
default=None,
help="Use CuDNN benchmark"
)
return parser
def build_args(parser):
parser.add_argument("--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--resume", type=str, default=None)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--train", type=int, default=None)
parser.add_argument("--valid", type=int, default=None)
parser.add_argument("--infer", type=int, default=None)
parser.add_argument("--vis", type=int, default=None)
boolean_flag(parser, "check", default=False)
boolean_flag(parser, "db", default=True)
parser.add_argument("--run-delay", type=int, default=1)
boolean_flag(parser, "daemon", default=True)
parser.add_argument("--sampler-id", type=int, default=0)
boolean_flag(parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend")
boolean_flag(parser, "benchmark", default=None, help="Use CuDNN benchmark")
return parser
def build_args(parser):
parser.add_argument(
"--in-csv",
type=Path,
help="Path to .csv with labels column",
required=True,
)
parser.add_argument(
"--in-label-column",
type=str,
help="Column to get labels",
required=False,
default="labels",
)
parser.add_argument(
"--in-npy",
type=Path,
help="Path to .npy with class logits",
required=True,
)
parser.add_argument(
"--out-thresholds",
type=Path,
help="Path to save .json with thresholds",
required=True,
)
parser.add_argument(
"--metric",
type=str,
help="Metric to use",
required=False,
choices=BINARY_PER_CLASS_METRICS,
default="roc_auc_score",
)
# parser.add_argument(
# "--ignore-label", type=int,
# required=False,
# default=None
# )
parser.add_argument(
"--num-splits", type=int, help="NUM_SPLITS", required=False, default=5
)
parser.add_argument(
"--num-repeats", type=int, help="NUM_REPEATS", required=False, default=1
)
parser.add_argument(
"--num-workers",
type=int,
help="CPU pool size",
required=False,
default=1,
)
utils.boolean_flag(parser, "verbose", default=False)
utils.boolean_flag(parser, "sigmoid", default=False)
return parser
def build_args(parser):
"""Constructs the command-line arguments for ``catalyst-data tag2label``.
Args:
parser: current parser
Returns:
updated parser
"""
parser.add_argument("--in-csv",
type=str,
default=None,
help="Path to data in `.csv`.")
parser.add_argument(
"--in-dir",
type=str,
default=None,
help="Path to directory with dataset" +
"or paths separated by commas for several datasets",
)
parser.add_argument(
"--out-dataset",
type=str,
default=None,
required=True,
help="Path to output dataframe",
)
parser.add_argument(
"--out-labeling",
type=str,
default=None,
required=True,
help="Path to output JSON",
)
parser.add_argument("--tag-column",
type=str,
default="tag",
help="Target column name")
parser.add_argument(
"--tag-delim",
type=str,
default=None,
help="Separator if you want to use several target columns",
)
utils.boolean_flag(
parser,
"recursive",
default=False,
help="Include subdirs in dataset",
)
return parser
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("--datapath", type=str, default=None)
parser.add_argument("--in-csv", type=str, required=True)
parser.add_argument("--out-dir", type=str, required=True)
parser.add_argument("--out-csv", type=str, required=True)
parser.add_argument("--n-cpu", type=int, default=None)
boolean_flag(parser, "verbose", default=False)
args = parser.parse_args()
return args
def build_args(parser):
parser.add_argument("--in-csv",
type=str,
default=None,
help="Path to data in `.csv`.")
parser.add_argument(
"--in-dir",
type=str,
default=None,
help="Path to directory with dataset"
"or paths separated by commas for several datasets",
)
parser.add_argument(
"--out-dataset",
type=str,
default=None,
required=True,
help="Path to output dataframe",
)
parser.add_argument(
"--out-labeling",
type=str,
default=None,
required=True,
help="Path to output JSON",
)
parser.add_argument("--tag-column",
type=str,
default="tag",
help="Target column name")
parser.add_argument(
"--tag-delim",
type=str,
default=None,
help="Separator if you want to use several target columns",
)
boolean_flag(
parser,
"recursive",
default=False,
help="Include subdirs in dataset",
)
return parser
def build_args(parser):
parser.add_argument("--host", type=str, default="127.0.0.1")
parser.add_argument("--port", type=int, default=12000)
parser.add_argument("--in-pkl",
"-P",
nargs="+",
metavar="PKL_PATH",
dest="in_pkl",
required=True)
parser.add_argument("--db",
type=str,
choices=["redis", "mongo"],
default=None,
required=True)
parser.add_argument("--min-reward", type=int, default=None)
utils.boolean_flag(parser,
"use-sqil",
default=False,
help="Use SQIL – 0 reward")
return parser
def build_args(parser: ArgumentParser):
"""Constructs the command-line arguments for ``catalyst-dl run``"""
parser.add_argument(
"--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True
)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--baselogdir", type=str, default=None)
parser.add_argument(
"-j",
"--num-workers",
default=None,
type=int,
help="number of data loading workers"
)
parser.add_argument(
"-b", "--batch-size", default=None, type=int, help="mini-batch size"
)
parser.add_argument(
"-e", "--num-epochs", default=None, type=int, help="number of epochs"
)
parser.add_argument(
"--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint"
)
parser.add_argument("--seed", type=int, default=42)
boolean_flag(parser, "verbose", default=None)
boolean_flag(parser, "check", default=None)
boolean_flag(
parser, "deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend"
)
boolean_flag(
parser, "benchmark",
default=None,
help="Use CuDNN benchmark"
)
return parser
def build_args(parser: ArgumentParser):
parser.add_argument("--config",
"--configs",
"-C",
nargs="+",
help="path to config/configs",
metavar="CONFIG_PATH",
dest="configs",
required=True)
parser.add_argument("--expdir", type=str, default=None)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--baselogdir", type=str, default=None)
parser.add_argument("-j",
"--num-workers",
default=None,
type=int,
help="number of data loading workers")
parser.add_argument("-b",
"--batch-size",
default=None,
type=int,
help="mini-batch size")
parser.add_argument("-e",
"--num-epochs",
default=None,
type=int,
help="number of epochs")
parser.add_argument("--resume",
default=None,
type=str,
metavar="PATH",
help="path to latest checkpoint")
parser.add_argument("--seed", type=int, default=42)
boolean_flag(parser, "verbose", default=False)
boolean_flag(parser, "check", default=False)
return parser
def build_args(parser):
parser.add_argument("--in-dir", type=Path)
parser.add_argument("--out-dir", type=Path)
parser.add_argument("--num-workers", "-j", type=int, default=1)
parser.add_argument("--extension", type=str, default="jpg")
parser.add_argument("--max-size", default=None, type=int)
boolean_flag(parser, "clear-exif", default=True)
boolean_flag(parser, "grayscale", default=False)
boolean_flag(parser, "expand-dims", default=True)
return parser
def build_args(parser):
"""
Constructs the command-line arguments for ``catalyst-data process-images``.
"""
parser.add_argument("--in-dir",
required=True,
type=Path,
help="Raw data folder path")
parser.add_argument(
"--out-dir",
required=True,
type=Path,
help="Processed images folder path",
)
parser.add_argument(
"--num-workers",
"-j",
default=1,
type=int,
help="Number of workers to parallel the processing",
)
parser.add_argument(
"--max-size",
default=None,
required=False,
type=int,
help="Output images size. E.g. 224, 448",
)
utils.boolean_flag(parser,
"clear-exif",
default=True,
help="Clear EXIF data")
utils.boolean_flag(parser,
"grayscale",
default=False,
help="Read images in grayscale")
utils.boolean_flag(
parser,
"expand-dims",
default=True,
help="Expand array shape for grayscale images",
)
return parser
def build_args(parser):
parser.add_argument("--in-dir",
required=True,
type=Path,
help="Raw data folder path")
parser.add_argument("--out-dir",
required=True,
type=Path,
help="Processed images folder path")
parser.add_argument("--num-workers",
"-j",
default=1,
type=int,
help="Number of workers to parallel the processing")
parser.add_argument("--extension",
default="jpg",
type=str,
help="Input images extension. JPG is default.")
parser.add_argument("--max-size",
default=None,
required=False,
type=int,
help="Output images size. E.g. 224, 448")
boolean_flag(parser, "clear-exif", default=True, help="Clear EXIF data")
boolean_flag(parser,
"grayscale",
default=False,
help="Read images in grayscale")
boolean_flag(parser,
"expand-dims",
default=True,
help="Expand array shape for grayscale images")
return parser
dl.CheckpointCallback(
self._logdir,
loader_key="valid",
metric_key="accuracy",
minimize=False,
save_n_best=1,
),
# "tqdm": dl.TqdmCallback(),
}
def handle_batch(self, batch):
x, y = batch
logits = self.model(x)
self.batch = {
"features": x,
"targets": y,
"logits": logits,
}
if __name__ == "__main__":
parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
parser.add_argument("--logdir", type=str, default=None)
parser.add_argument("--engine", type=str, choices=list(E2E.keys()))
utils.boolean_flag(parser, "sync-bn", default=False)
args, _ = parser.parse_known_args()
args.logdir = args.logdir or f"logs_resnet_{args.engine}_sbn{int(args.sync_bn)}".replace(
"-", "_")
runner = CustomRunner(args.logdir, args.engine, args.sync_bn)
runner.run()
def build_args(parser):
"""
Constructs the command-line arguments for
``catalyst-data image2embeddings``.
Args:
parser: parser
Returns:
modified parser
"""
parser.add_argument(
"--in-csv", type=str, dest="in_csv", help="Path to csv with photos"
)
parser.add_argument(
"--img-rootpath",
type=str,
dest="rootpath",
help="Path to photos directory",
)
parser.add_argument(
"--img-col",
type=str,
dest="img_col",
help="Column in table that contain image path",
)
parser.add_argument(
"--img-size",
type=int,
dest="img_size",
default=224,
help="Target size of images",
)
parser.add_argument(
"--out-npy",
type=str,
dest="out_npy",
required=True,
help="Path to output `.npy` file with embedded features",
)
parser.add_argument(
"--arch",
type=str,
dest="arch",
default="resnet18",
help="Neural network architecture",
)
parser.add_argument(
"--pooling",
type=str,
dest="pooling",
default="GlobalAvgPool2d",
help="Type of pooling to use",
)
parser.add_argument(
"--traced-model",
type=Path,
dest="traced_model",
default=None,
help="Path to pytorch traced model",
)
parser.add_argument(
"--num-workers",
type=int,
dest="num_workers",
help="Count of workers for dataloader",
default=0,
)
parser.add_argument(
"--batch-size",
type=int,
dest="batch_size",
help="Dataloader batch size",
default=32,
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Print additional information",
)
parser.add_argument("--seed", type=int, default=42)
utils.boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
utils.boolean_flag(
parser, "benchmark", default=None, help="Use CuDNN benchmark"
)
return parser
def build_args(parser):
"""
Constructs the command-line arguments for
``catalyst-data text2embeddings``.
Args:
parser: parser
Returns:
modified parser
"""
parser.add_argument("--in-csv",
type=str,
help="Path to csv with text",
required=True)
parser.add_argument("--txt-col",
type=str,
help="Column in table that contain text")
parser.add_argument(
"--in-huggingface",
type=str,
required=False,
help="model from huggingface hub",
)
required_path_to_model = True
for arg in sys.argv:
if "--in-huggingface" in arg:
required_path_to_model = False
if required_path_to_model:
parser.add_argument("--in-config",
type=Path,
required=required_path_to_model)
parser.add_argument("--in-model",
type=Path,
required=required_path_to_model)
parser.add_argument("--in-vocab",
type=Path,
required=required_path_to_model)
parser.add_argument(
"--out-prefix",
type=str,
required=True,
)
parser.add_argument("--max-length", type=int, default=512) # noqa: WPS432
utils.boolean_flag(parser, "mask-for-max-length", default=False)
utils.boolean_flag(parser, "output-hidden-states", default=False)
parser.add_argument(
"--bert-level",
type=int,
help="BERT features level to use",
default=None, # noqa: WPS432
)
utils.boolean_flag(parser, "strip", default=True)
utils.boolean_flag(parser, "lowercase", default=True)
utils.boolean_flag(parser, "remove-punctuation", default=True)
parser.add_argument("--pooling", type=str, default="avg")
parser.add_argument(
"--num-workers",
type=int,
dest="num_workers",
help="Count of workers for dataloader",
default=0,
)
parser.add_argument(
"--batch-size",
type=int,
dest="batch_size",
help="Dataloader batch size",
default=32, # noqa: WPS432
)
parser.add_argument(
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Print additional information",
)
parser.add_argument("--seed", type=int, default=42) # noqa: WPS432
utils.boolean_flag(
parser,
"deterministic",
default=None,
help="Deterministic mode if running in CuDNN backend",
)
utils.boolean_flag(parser,
"benchmark",
default=None,
help="Use CuDNN benchmark")
utils.boolean_flag(
parser,
"force-save",
default=None,
help="Force save `.npy` with np.save",
)
return parser
