def settings():
GPU_TRAIN = torch.cuda.is_available()
SEED = 2020
FP16 = True
NUM_CORES = multiprocessing.cpu_count()
BS = 8
if GPU_TRAIN:
CUDA_NAME = torch.cuda.get_device_name()
FP16 = True
if CUDA_NAME in ['Tesla K80', 'Tesla P4']:
BS = 32
else:
BS = 64
if FP16:
BS = int(BS * 2)
print(f'GPU: {CUDA_NAME}')
np.random.seed(SEED)
#random.seed(SEED)
set_global_seed(SEED)
prepare_cudnn()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(f'Number of cores CPU: {NUM_CORES}')
print(f'Batch size: {BS}')
print(torch.cuda.get_device_properties(device).total_memory / 2**20)
return NUM_CORES - 2, BS
def main(args, unknown_args):
args, config = parse_args_uargs(args, unknown_args)
set_global_seed(args.seed)
prepare_cudnn(args.deterministic, args.benchmark)
if args.logdir is not None:
os.makedirs(args.logdir, exist_ok=True)
dump_environment(config, args.logdir, args.configs)
if args.expdir is not None:
module = import_module(expdir=args.expdir) # noqa: F841
if args.logdir is not None:
dump_code(args.expdir, args.logdir)
env = ENVIRONMENTS.get_from_params(**config["environment"])
algorithm_name = config["algorithm"].pop("algorithm")
if algorithm_name in OFFPOLICY_ALGORITHMS_NAMES:
ALGORITHMS = OFFPOLICY_ALGORITHMS
trainer_fn = OffpolicyTrainer
sync_epoch = False
elif algorithm_name in ONPOLICY_ALGORITHMS_NAMES:
ALGORITHMS = ONPOLICY_ALGORITHMS
trainer_fn = OnpolicyTrainer
sync_epoch = True
else:
# @TODO: add registry for algorithms, trainers, samplers
raise NotImplementedError()
db_server = DATABASES.get_from_params(
**config.get("db", {}), sync_epoch=sync_epoch
)
algorithm_fn = ALGORITHMS.get(algorithm_name)
algorithm = algorithm_fn.prepare_for_trainer(env_spec=env, config=config)
if args.resume is not None:
checkpoint = utils.load_checkpoint(filepath=args.resume)
checkpoint = utils.any2device(checkpoint, utils.get_device())
algorithm.unpack_checkpoint(
checkpoint=checkpoint,
with_optimizer=False
)
monitoring_params = config.get("monitoring_params", None)
trainer = trainer_fn(
algorithm=algorithm,
env_spec=env,
db_server=db_server,
logdir=args.logdir,
monitoring_params=monitoring_params,
**config["trainer"],
)
trainer.run()
def main(args, unknown_args):
args, config = parse_args_uargs(args, unknown_args)
set_global_seed(args.seed)
prepare_cudnn(args.deterministic, args.benchmark)
Experiment, Runner = import_experiment_and_runner(Path(args.expdir))
experiment = Experiment(config)
runner = Runner()
if experiment.logdir is not None:
dump_config(config, experiment.logdir, args.configs)
dump_code(args.expdir, experiment.logdir)
runner.run_experiment(experiment, check=args.check)
def main(args, unknown_args):
"""Run the ``catalyst-dl run`` script"""
args, config = parse_args_uargs(args, unknown_args)
set_global_seed(args.seed)
prepare_cudnn(args.deterministic, args.benchmark)
Experiment, Runner = import_experiment_and_runner(Path(args.expdir))
runner_params = config.pop("runner_params", {}) or {}
experiment = Experiment(config)
runner = Runner(**runner_params)
if experiment.logdir is not None:
dump_environment(config, experiment.logdir, args.configs)
dump_code(args.expdir, experiment.logdir)
check_run = safitty.get(config, "args", "check", default=False)
runner.run_experiment(experiment, check=check_run)
def main(args, _=None):
"""Run the ``catalyst-data image2embeddings`` script."""
global IMG_SIZE
utils.set_global_seed(args.seed)
utils.prepare_cudnn(args.deterministic, args.benchmark)
IMG_SIZE = (args.img_size, args.img_size) # noqa: WPS442
if args.traced_model is not None:
device = utils.get_device()
model = torch.jit.load(str(args.traced_model), map_location=device)
else:
model = ResnetEncoder(arch=args.arch, pooling=args.pooling)
model = model.eval()
model, _, _, _, device = utils.process_components(model=model)
df = pd.read_csv(args.in_csv)
df = df.reset_index().drop("index", axis=1)
df = list(df.to_dict("index").values())
open_fn = ImageReader(
input_key=args.img_col, output_key="image", rootpath=args.rootpath
)
dataloader = utils.get_loader(
df,
open_fn,
batch_size=args.batch_size,
num_workers=args.num_workers,
dict_transform=dict_transformer,
)
features = []
dataloader = tqdm(dataloader) if args.verbose else dataloader
with torch.no_grad():
for batch in dataloader:
batch_features = model(batch["image"].to(device))
batch_features = batch_features.cpu().detach().numpy()
features.append(batch_features)
features = np.concatenate(features, axis=0)
np.save(args.out_npy, features)
def setup_runtime(cfg_env: DictConfig):
"""
Setup runtime environment.
Runtime options:
["cuda", "cuda:0", "cuda:<index>", "cpu", ""]
Args:
cfg_env (dict): Configuration
Returns:
None
"""
runtime: str = cfg_env.runtime
runtime_name, runtime_devices = \
runtime.split(":") if ":" in runtime else [runtime, ""]
if runtime_name == "cuda" and runtime_devices:
os.environ["CUDA_VISIBLE_DEVICES"] = f"{runtime_devices}"
logger.info(f"[Environment] Configuration: CUDA_VISIBLE_DEVICES="
f"{os.environ['CUDA_VISIBLE_DEVICES']}")
elif runtime_name == "cpu":
os.environ["CUDA_VISIBLE_DEVICES"] = ""
logger.info(f"[Environment] Configuration: CUDA_VISIBLE_DEVICES="
f"{os.environ['CUDA_VISIBLE_DEVICES']}")
from catalyst.utils import set_global_seed, prepare_cudnn, get_device
seed: int = cfg_env.seed
set_global_seed(seed)
logger.info(f"[Environment] Configuration. Seed: {seed}")
prepare_cudnn(deterministic=True, benchmark=False)
logger.info(f"[Environment] Configuration. CUDNN: "
f"deterministic=True, benchmark=False")
device = get_device()
logger.info(f"[Environment] Runtime: {device}")
return device
def make_classifier():
# Set seeds for reproducibility
set_global_seed(SEED)
prepare_cudnn(deterministic=True)
np.random.seed(SEED)
torch.manual_seed(SEED)
model = BertForSequenceClassification(PRETRAINED_MODEL_NAME, NUM_LABELS)
model.to(device)
print(f'Loaded model: {PRETRAINED_MODEL_NAME}')
print(f'Trainable parameters: {model.n_trainable()}')
criterion = model.configure_loss()
optimizer = model.configure_optimizers(1)
scheduler = model.configure_scheduler(optimizer)
return {
'criterion': criterion,
'optimizer': optimizer,
'scheduler': scheduler,
'model': model,
# 'epoch': 0
}
test_filenames)}, columns=['ImageFileName'])
test_dataset = DataLoader(
ALASKATestData(test_df, augmentations=albu.Compose([
# albu.CenterCrop()
albu.Normalize(),
ToTensorV2()
])
),
batch_size=1, shuffle=False, num_workers=args.nw)
print(len(train_data))
print(len(val_data))
SEED = 2020
utils.set_global_seed(SEED)
utils.prepare_cudnn(deterministic=True)
loaders = {'train': train_data,
'valid': val_data}
criterion = nn.CrossEntropyLoss()
model = ENet('efficientnet-b0')
print(model)
optimizer = Lookahead(RAdam(
model.parameters(), lr=args.lr, weight_decay=args.wd))
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer, factor=0.25, patience=3)
num_epochs = args.e
logdir = "./logs/effnet-b0"
fp16_params = None # dict(opt_level="O1")
runner = SupervisedRunner(device='cuda')
def main(args, unknown_args):
args, config = parse_args_uargs(args, unknown_args)
set_global_seed(args.seed)
prepare_cudnn(args.deterministic, args.benchmark)
args.vis = args.vis or 0
args.infer = args.infer or 0
args.valid = args.valid or 0
args.train = args.train or 0
if args.expdir is not None:
module = import_module(expdir=args.expdir) # noqa: F841
environment_name = config["environment"].pop("environment")
environment_fn = ENVIRONMENTS.get(environment_name)
algorithm_name = config["algorithm"].pop("algorithm")
if algorithm_name in OFFPOLICY_ALGORITHMS_NAMES:
ALGORITHMS = OFFPOLICY_ALGORITHMS
sync_epoch = False
elif algorithm_name in ONPOLICY_ALGORITHMS_NAMES:
ALGORITHMS = ONPOLICY_ALGORITHMS
sync_epoch = True
else:
raise NotImplementedError()
algorithm_fn = ALGORITHMS.get(algorithm_name)
processes = []
sampler_id = args.sampler_id
def on_exit():
for p in processes:
p.terminate()
atexit.register(on_exit)
params = dict(
seed=args.seed,
logdir=args.logdir,
algorithm_fn=algorithm_fn,
environment_fn=environment_fn,
config=config,
resume=args.resume,
db=args.db,
sync_epoch=sync_epoch
)
if args.check:
mode = "train"
mode = "valid" if (args.valid is not None and args.valid > 0) else mode
mode = "infer" if (args.infer is not None and args.infer > 0) else mode
params_ = dict(
visualize=(args.vis is not None and args.vis > 0),
mode=mode,
id=sampler_id
)
run_sampler(**params, **params_)
return
for i in range(args.vis):
params_ = dict(
visualize=True, mode="infer", id=sampler_id, exploration_power=0.0
)
p = mp.Process(
target=run_sampler,
kwargs=dict(**params, **params_),
daemon=args.daemon,
)
p.start()
processes.append(p)
sampler_id += 1
time.sleep(args.run_delay)
for i in range(args.infer):
params_ = dict(
visualize=False,
mode="infer",
id=sampler_id,
exploration_power=0.0
)
p = mp.Process(
target=run_sampler,
kwargs=dict(**params, **params_),
daemon=args.daemon,
)
p.start()
processes.append(p)
sampler_id += 1
time.sleep(args.run_delay)
for i in range(args.valid):
params_ = dict(
visualize=False,
mode="valid",
id=sampler_id,
exploration_power=0.0
)
p = mp.Process(
target=run_sampler,
kwargs=dict(**params, **params_),
daemon=args.daemon,
)
p.start()
processes.append(p)
sampler_id += 1
time.sleep(args.run_delay)
for i in range(1, args.train + 1):
exploration_power = i / args.train
params_ = dict(
visualize=False,
mode="train",
id=sampler_id,
exploration_power=exploration_power
)
p = mp.Process(
target=run_sampler,
kwargs=dict(**params, **params_),
daemon=args.daemon,
)
p.start()
processes.append(p)
sampler_id += 1
time.sleep(args.run_delay)
for p in processes:
p.join()
parser.add_argument("--train", help="train", type=bool, default=False)
parser.add_argument("--make_prediction", help="to make prediction", type=bool, default=False)
parser.add_argument("--preload", help="save processed data", type=bool, default=False)
parser.add_argument("--separate_decoder", help="number of epochs", type=bool, default=False)
parser.add_argument("--multigpu", help="use multi-gpu", type=bool, default=False)
parser.add_argument("--lookahead", help="use lookahead", type=bool, default=False)
args, unknown = parser.parse_known_args()
# args.train = False
args.optimize_postprocess = False
print(args)
if args.task == 'classification':
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
set_global_seed(args.seed)
prepare_cudnn(deterministic=True)
sub_name = f'Model_{args.task}_{args.model_type}_{args.encoder}_bs_{args.bs}_{str(datetime.datetime.now().date())}'
logdir = f"./logs/{sub_name}" if args.logdir is None else args.logdir
preprocessing_fn = smp.encoders.get_preprocessing_fn(args.encoder, args.encoder_weights)
loaders = prepare_loaders(path=args.path, bs=args.bs,
num_workers=args.num_workers, preprocessing_fn=preprocessing_fn, preload=args.preload,
image_size=(args.height, args.width), augmentation=args.augmentation, task=args.task)
test_loader = loaders['test']
del loaders['test']
model = get_model(model_type=args.segm_type, encoder=args.encoder, encoder_weights=args.encoder_weights,
activation=None, task=args.task)
optimizer = get_optimizer(optimizer=args.optimizer, lookahead=args.lookahead, model=model,
def main():
# Enable argument parsing for file paths
args = vars(get_args())
train_images_path = args["train_images"]
train_masks_path = args["train_masks"]
test_images_path = args["test_images"]
test_masks_path = args["test_masks"]
# print out yaml file configuration
dir_path = os.path.dirname(os.path.realpath(__file__))
yaml_path = os.path.join(dir_path, "config/igvc.yaml")
ARCH = yaml.safe_load(open(yaml_path, "r"))
# Set a seed for reproducibility
utils.set_global_seed(ARCH["train"]["seed"])
utils.prepare_cudnn(deterministic=ARCH["train"]["cudnn"])
# Set up U-Net with pretrained EfficientNet backbone
model = smp.Unet(
encoder_name=ARCH["encoder"]["name"],
encoder_weights=ARCH["encoder"]["weight"],
classes=ARCH["train"]["classes"],
activation=ARCH["encoder"]["activation"],
)
# Get Torch loaders
loaders = get_loaders(
images=np.load(train_images_path),
masks=np.load(train_masks_path),
image_arr_path=train_images_path,
mask_arr_path=train_masks_path,
random_state=ARCH["train"]["random_state"],
valid_size=ARCH["train"]["valid_size"],
batch_size=ARCH["train"]["batch_size"],
num_workers=ARCH["train"]["num_workers"],
)
# Optimize for cross entropy using Adam
criterion = {
"CE": CrossentropyND(),
}
optimizer = AdamW(
model.parameters(),
lr=ARCH["train"]["lr"],
betas=(ARCH["train"]["betas_min"], ARCH["train"]["betas_max"]),
eps=float(ARCH["train"]["eps"]),
weight_decay=ARCH["train"]["w_decay"],
amsgrad=ARCH["train"]["amsgrad"],
)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
factor=ARCH["train"]["optim_factor"],
patience=ARCH["train"]["optim_patience"],
)
device = utils.get_device()
print("Using device: {}".format(device))
print(f"torch: {torch.__version__}, catalyst: {catalyst.__version__}")
runner = SupervisedRunner(device=device,
input_key="image",
input_target_key="mask")
# Use Catalyst callbacks for metric calculations during training
callbacks = [
CriterionCallback(input_key="mask", prefix="loss", criterion_key="CE"),
MulticlassDiceMetricCallback(input_key="mask"),
]
# Train and print model training logs
runner.train(
model=model,
criterion=criterion,
optimizer=optimizer,
scheduler=scheduler,
loaders=loaders,
callbacks=callbacks,
logdir=ARCH["train"]["logdir"],
num_epochs=ARCH["train"]["epochs"],
main_metric="loss",
minimize_metric=ARCH["train"]["minimize_metric"],
fp16=ARCH["train"]["fp16"],
verbose=ARCH["train"]["verbose"],
)
# Test model on test dataset
test_data = SegmentationDataset(test_images_path, test_masks_path)
infer_loader = DataLoader(
test_data,
batch_size=ARCH["test"]["batch_size"],
shuffle=ARCH["test"]["shuffle"],
num_workers=ARCH["test"]["num_workers"],
)
# Get model predictions on test dataset
predictions = np.vstack(
list(
map(
lambda x: x["logits"].cpu().numpy(),
runner.predict_loader(
loader=infer_loader,
resume=f"content/full_model2/checkpoints/best.pth",
),
)))
save_result(predictions, test_data)
from torch import nn
from torch.optim import Adam
from torch.optim.lr_scheduler import ReduceLROnPlateau
from utils.dataset import get_train_val_dataloaders
from utils.callbacks import DiceCallback as MyDiceCallbak, IouCallback as MyIouCallback
from utils.coord_conv import CoordConv
from catalyst.dl import SupervisedRunner, DiceCallback, IouCallback
from catalyst.utils import set_global_seed, prepare_cudnn
import segmentation_models_pytorch as smp
prepare_cudnn(True, True)
set_global_seed(0)
class Model(nn.Module):
def __init__(self, encoder):
super().__init__()
self.coord_conv = CoordConv(3, 3, True, kernel_size=3, padding=1)
self.coord_conv_decoder = CoordConv(16,
16,
True,
kernel_size=3,
padding=1)
self.model = smp.Unet(encoder,
encoder_weights='imagenet',
classes=4,
activation=None)
def seed_all(SEED):
set_global_seed(SEED)
prepare_cudnn(deterministic=True)
def main(args, _=None):
"""Run the ``catalyst-data text2embeddings`` script."""
batch_size = args.batch_size
num_workers = args.num_workers
max_length = args.max_length
pooling_groups = args.pooling.split(",")
bert_level = args.bert_level
if bert_level is not None:
assert (args.output_hidden_states
), "You need hidden states output for level specification"
utils.set_global_seed(args.seed)
utils.prepare_cudnn(args.deterministic, args.benchmark)
if getattr(args, "in_huggingface", False):
model_config = BertConfig.from_pretrained(args.in_huggingface)
model_config.output_hidden_states = args.output_hidden_states
model = BertModel.from_pretrained(args.in_huggingface,
config=model_config)
tokenizer = BertTokenizer.from_pretrained(args.in_huggingface)
else:
model_config = BertConfig.from_pretrained(args.in_config)
model_config.output_hidden_states = args.output_hidden_states
model = BertModel(config=model_config)
tokenizer = BertTokenizer.from_pretrained(args.in_vocab)
if getattr(args, "in_model", None) is not None:
checkpoint = utils.load_checkpoint(args.in_model)
checkpoint = {"model_state_dict": checkpoint}
utils.unpack_checkpoint(checkpoint=checkpoint, model=model)
model = model.eval()
model, _, _, _, device = utils.process_components(model=model)
df = pd.read_csv(args.in_csv)
df = df.dropna(subset=[args.txt_col])
df.to_csv(f"{args.out_prefix}.df.csv", index=False)
df = df.reset_index().drop("index", axis=1)
df = list(df.to_dict("index").values())
num_samples = len(df)
open_fn = LambdaReader(
input_key=args.txt_col,
output_key=None,
lambda_fn=partial(
tokenize_text,
strip=args.strip,
lowercase=args.lowercase,
remove_punctuation=args.remove_punctuation,
),
tokenizer=tokenizer,
max_length=max_length,
)
dataloader = utils.get_loader(
df,
open_fn,
batch_size=batch_size,
num_workers=num_workers,
)
features = {}
dataloader = tqdm(dataloader) if args.verbose else dataloader
with torch.no_grad():
for idx, batch_input in enumerate(dataloader):
batch_input = utils.any2device(batch_input, device)
batch_output = model(**batch_input)
mask = (batch_input["attention_mask"].unsqueeze(-1)
if args.mask_for_max_length else None)
if utils.check_ddp_wrapped(model):
# using several gpu
hidden_size = model.module.config.hidden_size
hidden_states = model.module.config.output_hidden_states
else:
# using cpu or one gpu
hidden_size = model.config.hidden_size
hidden_states = model.config.output_hidden_states
batch_features = process_bert_output(
bert_output=batch_output,
hidden_size=hidden_size,
output_hidden_states=hidden_states,
pooling_groups=pooling_groups,
mask=mask,
)
# create storage based on network output
if idx == 0:
for layer_name, layer_value in batch_features.items():
if bert_level is not None and bert_level != layer_name:
continue
layer_name = (layer_name if isinstance(layer_name, str)
else f"{layer_name:02d}")
_, embedding_size = layer_value.shape
features[layer_name] = np.memmap(
f"{args.out_prefix}.{layer_name}.npy",
dtype=np.float32,
mode="w+",
shape=(num_samples, embedding_size),
)
indices = np.arange(idx * batch_size,
min((idx + 1) * batch_size, num_samples))
for layer_name2, layer_value2 in batch_features.items():
if bert_level is not None and bert_level != layer_name2:
continue
layer_name2 = (layer_name2 if isinstance(layer_name2, str) else
f"{layer_name2:02d}")
features[layer_name2][indices] = _detach(layer_value2)
if args.force_save:
for key, mmap in features.items():
mmap.flush()
np.save(f"{args.out_prefix}.{key}.force.npy",
mmap,
allow_pickle=False)
