def get_test_dataset(cfg: DictConfig):
"""
Get test dataset
Args:
cfg:
Returns:
"""
test_img_dir = f'{cfg.data.folder_path}/test'
valid_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['valid']['augs']
]
valid_bbox_params = OmegaConf.to_container(
(cfg['augmentation']['valid']['bbox_params']))
valid_augs = A.Compose(valid_augs_list, bbox_params=valid_bbox_params)
dataset_class = load_obj(cfg.dataset.class_name)
test_dataset = dataset_class(None, 'test', test_img_dir, cfg, valid_augs)
return test_dataset
def configure_optimizers(self):
if 'decoder_lr' in self.cfg.optimizer.params.keys():
params = [
{
'params': self.model.decoder.parameters(),
'lr': self.cfg.optimizer.params.lr
},
{
'params': self.model.encoder.parameters(),
'lr': self.cfg.optimizer.params.decoder_lr
},
]
optimizer = load_obj(self.cfg.optimizer.class_name)(params)
else:
optimizer = load_obj(self.cfg.optimizer.class_name)(
self.model.parameters(), **self.cfg.optimizer.params)
scheduler = load_obj(self.cfg.scheduler.class_name)(
optimizer, **self.cfg.scheduler.params)
return [optimizer], [{
'scheduler': scheduler,
'interval': self.cfg.scheduler.step,
'monitor': self.cfg.scheduler.monitor
}]
def get_m5model(cfg: DictConfig) -> Any:
"""
Get model
Args:
cfg: config
Returns:
initialized model
"""
backcast_length = cfg.dataset.backcast_length
forecast_length = cfg.dataset.forecast_length
f_b_dim = (forecast_length, backcast_length)
# collect stack parameters.
model_dict: Dict[str, list] = {k: [] for k in cfg.model.blocks[0].keys()}
for block in cfg.model.blocks:
for k, v in block.items():
if type(v) == str:
v = load_obj(v)
model_dict[k].append(v)
criterion = load_obj(cfg.loss.class_name)(**cfg.loss.params)
net = load_obj(cfg.model.class_name)
net = net(f_b_dim=f_b_dim, criterion=criterion, **model_dict)
return net
def configure_optimizers(self):
optimizer = load_obj(self.cfg.optimizer.class_name)(self.net.parameters(), **self.cfg.optimizer.params)
scheduler = load_obj(self.cfg.scheduler.class_name)(optimizer, **self.cfg.scheduler.params)
return (
[optimizer],
[{'scheduler': scheduler, 'interval': self.cfg.scheduler.step, 'monitor': self.cfg.scheduler.monitor}],
)
def __init__(self, cfg: DictConfig) -> None:
"""
Model class.
Args:
cfg: main config
"""
super().__init__()
self.encoder = load_obj(
cfg.model.encoder.class_name)(**cfg.model.encoder.params)
self.decoder = load_obj(cfg.model.decoder.class_name)(
output_dimension=self.encoder.output_dimension,
**cfg.model.decoder.params)
self.loss = load_obj(cfg.loss.class_name)()
def get_datasets(cfg: DictConfig) -> dict:
"""
Get datases for modelling
Args:
cfg: config
Returns:
dict with datasets
"""
train_df = pd.read_csv(
f'{cfg.data.folder_path}/data/sales_train_evaluation.csv')
train_fold_df = train_df.iloc[:, :-28]
valid_fold_df = train_df.iloc[:, -28:].copy()
backcast_length = cfg.dataset.backcast_length
# forecast_length = cfg.dataset.forecast_length
# train dataset
dataset_class = load_obj(cfg.dataset.class_name)
# get only useful columns
train_data = pd.concat(
[train_fold_df.iloc[:, 0], train_fold_df.iloc[:, 6:-1]], axis=1)
train_dataset = dataset_class(df=train_data, mode='train', cfg=cfg)
valid_data = pd.concat([
train_fold_df.iloc[:, 0],
train_fold_df.iloc[:, -backcast_length - 1:-1], valid_fold_df
],
axis=1)
valid_dataset = dataset_class(df=valid_data, mode='valid', cfg=cfg)
return {'train': train_dataset, 'valid': valid_dataset}
def __init__(self, hparams: Dict[str, float], cfg: DictConfig):
super(LitM5NBeats, self).__init__()
self.cfg = cfg
self.hparams: Dict[str, float] = hparams
self.net = get_m5model(self.cfg)
self.hparams['n_params'] = sum(p.numel() for p in self.net.parameters())
self.criterion = load_obj(self.cfg.loss.class_name)(**self.cfg.loss.params)
self._prepare_evaluator()
def __init__(self, cfg: DictConfig):
super().__init__()
self.model = load_obj(cfg.model.backbone.class_name)
self.model = self.model(**cfg.model.backbone.params)
# get number of input features for the classifier
self.output_dimension = self.model.roi_heads.box_predictor.cls_score.in_features
def get_training_datasets(cfg: DictConfig) -> Dict:
"""
Get datases for modelling
Args:
cfg: config
Returns:
dict with datasets
"""
train = pd.read_csv(f'{cfg.data.folder_path}/train.csv')
train[['x', 'y', 'w', 'h']] = pd.DataFrame(
np.stack(train['bbox'].apply(lambda x: ast.literal_eval(x)))).astype(
np.float32)
# precalculate some values
train['x1'] = train['x'] + train['w']
train['y1'] = train['y'] + train['h']
train['area'] = train['w'] * train['h']
train_ids, valid_ids = train_test_split(train['image_id'].unique(),
test_size=0.1,
random_state=cfg.training.seed)
# for fast training
if cfg.training.debug:
train_ids = train_ids[:10]
valid_ids = valid_ids[:10]
train_df = train.loc[train['image_id'].isin(train_ids)]
valid_df = train.loc[train['image_id'].isin(valid_ids)]
train_img_dir = f'{cfg.data.folder_path}/train'
# train dataset
dataset_class = load_obj(cfg.dataset.class_name)
# initialize augmentations
train_augs = load_augs(cfg['augmentation']['train']['augs'])
valid_augs = load_augs(cfg['augmentation']['valid']['augs'])
train_dataset = dataset_class(dataframe=train_df,
mode='train',
image_dir=train_img_dir,
cfg=cfg,
transforms=train_augs)
valid_dataset = dataset_class(dataframe=valid_df,
mode='valid',
image_dir=train_img_dir,
cfg=cfg,
transforms=valid_augs)
return {'train': train_dataset, 'valid': valid_dataset}
def get_training_datasets(cfg: DictConfig) -> dict:
data_dir = f'{cfg.data.folder_path}'
data_ids = [
fname.split('.')[0] for fname in os.listdir(f'{data_dir}/img_train')
]
train_ids, valid_ids = train_test_split(data_ids,
test_size=0.1,
random_state=cfg.training.seed)
# for fast training
if cfg.training.debug:
train_ids = train_ids[:10]
valid_ids = valid_ids[:10]
# dataset
dataset_class = load_obj(cfg.dataset.class_name)
# initialize augmentations
train_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['train']['augs']
]
train_augs = A.Compose(train_augs_list)
valid_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['valid']['augs']
]
valid_augs = A.Compose(valid_augs_list)
train_dataset = dataset_class(train_ids, 'train', data_dir, cfg,
train_augs)
valid_dataset = dataset_class(valid_ids, 'valid', data_dir, cfg,
valid_augs)
return {'train': train_dataset, 'valid': valid_dataset}
def get_wheat_model(cfg: DictConfig = None):
"""
Get model
Args:
cfg: config
Returns:
"""
model = load_obj(cfg.model.backbone.class_name)
model = model(**cfg.model.backbone.params)
# get number of input features for the classifier
in_features = model.roi_heads.box_predictor.cls_score.in_features
head = load_obj(cfg.model.head.class_name)
# replace the pre-trained head with a new one
model.roi_heads.box_predictor = head(in_features, cfg.model.head.params.num_classes)
return model
def get_unet_model(cfg: DictConfig = None):
"""
Get model
Args:
cfg: config
Returns:
"""
model = load_obj(cfg.model.backbone.class_name)
model = model(**cfg.model.backbone.params)
return model
def get_test_dataset(cfg: DictConfig):
"""
Get test dataset
Args:
cfg:
Returns:
"""
test_img_dir = f'{cfg.data.folder_path}/data55401/img_testA'
valid_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['valid']['augs']
]
valid_augs = A.Compose(valid_augs_list)
dataset_class = load_obj(cfg.dataset.class_name)
test_dataset = dataset_class(None, 'test', test_img_dir, cfg, valid_augs)
return test_dataset
def load_augs(cfg: DictConfig) -> A.Compose:
"""
Load albumentations
Args:
cfg:
Returns:
compose object
"""
augs = []
for a in cfg:
if a['class_name'] == 'albumentations.OneOf':
small_augs = []
for small_aug in a['params']:
# yaml can't contain tuples, so we need to convert manually
params = {
k: (v if type(v) != omegaconf.listconfig.ListConfig else
tuple(v))
for k, v in small_aug['params'].items()
}
aug = load_obj(small_aug['class_name'])(**params)
small_augs.append(aug)
aug = load_obj(a['class_name'])(small_augs)
augs.append(aug)
else:
params = {
k:
(v if type(v) != omegaconf.listconfig.ListConfig else tuple(v))
for k, v in a['params'].items()
}
aug = load_obj(a['class_name'])(**params)
augs.append(aug)
return A.Compose(augs)
def get_test_dataset(cfg: DictConfig) -> object:
"""
Get test dataset
Args:
cfg:
Returns:
test dataset
"""
test_img_dir = f'{cfg.data.folder_path}/test'
valid_augs = load_augs(cfg['augmentation']['valid']['augs'])
dataset_class = load_obj(cfg.dataset.class_name)
test_dataset = dataset_class(dataframe=None,
mode='test',
image_dir=test_img_dir,
cfg=cfg,
transforms=valid_augs)
return test_dataset
def get_training_datasets(cfg: DictConfig) -> dict:
"""
Get datases for modelling
Args:
cfg: config
Returns:
"""
train = pd.read_csv(f'{cfg.data.folder_path}/train.csv')
train[['x', 'y', 'w', 'h']] = pd.DataFrame(
np.stack(train['bbox'].apply(lambda x: ast.literal_eval(x)))).astype(
np.float32)
# precalculate some values
train['x1'] = train['x'] + train['w']
train['y1'] = train['y'] + train['h']
train['area'] = train['w'] * train['h']
train_ids, valid_ids = train_test_split(train['image_id'].unique(),
test_size=0.1,
random_state=cfg.training.seed)
# for fast training
if cfg.training.debug:
train_ids = train_ids[:10]
valid_ids = valid_ids[:10]
train_df = train.loc[train['image_id'].isin(train_ids)]
valid_df = train.loc[train['image_id'].isin(valid_ids)]
train_img_dir = f'{cfg.data.folder_path}/train'
# train dataset
dataset_class = load_obj(cfg.dataset.class_name)
# initialize augmentations
train_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['train']['augs']
]
train_bbox_params = OmegaConf.to_container(
(cfg['augmentation']['train']['bbox_params']))
train_augs = A.Compose(train_augs_list, bbox_params=train_bbox_params)
valid_augs_list = [
load_obj(i['class_name'])(**i['params'])
for i in cfg['augmentation']['valid']['augs']
]
valid_bbox_params = OmegaConf.to_container(
(cfg['augmentation']['valid']['bbox_params']))
valid_augs = A.Compose(valid_augs_list, bbox_params=valid_bbox_params)
train_dataset = dataset_class(train_df, 'train', train_img_dir, cfg,
train_augs)
valid_dataset = dataset_class(valid_df, 'valid', train_img_dir, cfg,
valid_augs)
return {'train': train_dataset, 'valid': valid_dataset}
def configure_optimizers(self):
optimizer = load_obj(self.hparams.optimizer.class_name)(self.model.parameters(),
**self.hparams.optimizer.params)
scheduler = load_obj(self.hparams.scheduler.class_name)(optimizer, **self.hparams.scheduler.params)
return [optimizer], [scheduler]