def get_all_stats(data_dir, folds, size=224):
"""
Function that creates a dictionary with the RGB means and stds.
"""
# Reproducibility
myutils.myseed(seed=42)
stats_dict = {}
for fold in range(1, folds + 1):
path = os.path.join(data_dir, f'train{fold}.csv')
train = pd.read_csv(path)
mean, std = myutils.get_stats(train, size)
stats_dict[f'mean{fold}'] = mean
stats_dict[f'std{fold}'] = std
# Save stats_dict to a .json file
with open('dicts/stats_dict.json', 'w') as f:
f.write(json.dumps(stats_dict))
logging_data_process.info('Saved: dicts/stats_dict.json')
for fold in range(args.fold_start, args.folds + 1):
# Load data from .csv files
tname = os.path.join(args.data_dir, 'train.csv')
vname = os.path.join(args.data_dir, 'val.csv')
if args.folds > 1:
# Load data from .csv files
tname = os.path.join(args.data_dir, f'train{fold}.csv')
vname = os.path.join(args.data_dir, f'val{fold}.csv')
train = pd.read_csv(tname)
print(tname)
val = pd.read_csv(vname)
dfs['train'] = train
dfs['val'] = val
mean, std = myutils.get_stats(train, params.size)
logging_process.info(
f'Model: {args.model_dir}\tFold: {fold}\tTrain: {tname}\tMean: {mean}\tStd: {std}'
)
# NETWORK SETTINGS
# Data
loaders = myutils.get_module(args.net_dir, 'loaders')
dataloaders, dataset_sizes = loaders.get_loaders(
dfs,
mean,
std,
size=params.size,
batch_size=params.batch_size,
num_workers=params.num_workers)
# Net