def _load_analytics_csv(dataset_folder, inmem, workers):
"""
This function loads the analytics.csv file and attempts creating it, if it is missing
Parameters
----------
dataset_folder : string
Path string that points to the three folder train/val/test. Example: ~/../../data/svhn
inmem : boolean
Flag: if False, the dataset is loaded in an online fashion i.e. only file names are stored and images are loaded
on demand. This is slower than storing everything in memory.
workers : int
Number of workers to use for the mean/std computation
Returns
-------
file
The csv file
"""
# If analytics.csv file not present, run the analytics on the dataset
if not os.path.exists(os.path.join(dataset_folder, "analytics.csv")):
logging.warning('Missing analytics.csv file for dataset located at {}'.format(dataset_folder))
try:
logging.warning('Attempt creating analytics.csv file for dataset located at {}'.format(dataset_folder))
compute_mean_std(dataset_folder=dataset_folder, inmem=inmem, workers=workers)
logging.warning('Created analytics.csv file for dataset located at {} '.format(dataset_folder))
except:
logging.error('Creation of analytics.csv failed.')
sys.exit(-1)
# Loads the analytics csv
return pd.read_csv(os.path.join(dataset_folder, "analytics.csv"), header=None)
def _load_analytics_csv(dataset_folder, inmem, workers, runner_class,
**kwargs):
"""
This function loads the analytics.csv file and attempts creating it, if it is missing
Parameters
----------
dataset_folder : string
Path string that points to the three folder train/val/test. Example: ~/../../data/svhn
inmem : boolean
Flag: if False, the dataset is loaded in an online fashion i.e. only file names are stored and images are loaded
on demand. This is slower than storing everything in memory.
workers : int
Number of workers to use for the mean/std computation
runner_class: string
specifies the runner class (mean and std have to be computed differently for the semantic segmentation)
Returns
-------
file
The csv file
"""
# If analytics.csv file not present, run the analytics on the dataset
if not os.path.exists(os.path.join(dataset_folder, "analytics.csv")):
logging.warning(
'Missing analytics.csv file for dataset located at {}'.format(
dataset_folder))
try:
logging.warning(
'Attempt creating analytics.csv file for dataset located at {}'
.format(dataset_folder))
if runner_class is not None and 'segmentation' in runner_class:
compute_mean_std_segmentation(
dataset_folder=dataset_folder,
inmem=inmem,
workers=workers,
filter_boundaries=True
if 'divahisdb' in runner_class else False)
else:
compute_mean_std(dataset_folder=dataset_folder,
inmem=inmem,
workers=workers)
logging.warning(
'Created analytics.csv file for dataset located at {} '.format(
dataset_folder))
except:
logging.error('Creation of analytics.csv failed.')
sys.exit(-1)
# Loads the analytics csv
return pd.read_csv(os.path.join(dataset_folder, "analytics.csv"),
header=None)