def __init__(self, file, fps, resolution):
self.file = bd.process_path(file)
self.extension = f'.{file.split(".")[-1]}'
self.writer = None
self.context_count = 0
self.fourcc = Video.LDR_VID_EXT[self.extension]['fourcc']
self.fps = float(fps)
self.resolution = resolution
directory = os.path.dirname(file)
bd.process_path(directory, create=True)
def __new__(cls, filename, fps=None, resolution=None):
if not isinstance(filename, str):
raise RuntimeError(
f'Invalid filename, expected str type, got: {type(filename)}')
filename = bd.process_path(filename)
all_extensions = (list(Video.HDR_VID_EXT.keys()) +
list(Video.HDR_IMG_EXT.keys()) +
list(Video.LDR_IMG_EXT.keys()) +
list(Video.LDR_VID_EXT.keys()))
extension = '.' + filename.split(".")[-1]
if extension not in all_extensions:
raise RuntimeError(f'Unsupported extension: {extension}')
is_ldr_video_file = extension in Video.LDR_VID_EXT
_id = (filename, extension)
if _id in Video._VIDEOS:
return Video._VIDEOS[_id]
self = super().__new__(cls)
Video._VIDEOS[_id] = self
self.is_ldr_video_file = is_ldr_video_file
self.extension = extension
self.filename = filename
self.dirname = os.path.dirname(filename)
self.fps = fps
self.resolution = resolution
return self
def __init__(self,
name,
fields=None,
directory=".",
delimiter=',',
resume=True):
if fields is None:
fields = ['value']
_check_valid_fields(fields)
if not name.endswith('.csv'):
name = name + '.csv'
self.directory = bd.process_path(directory, True)
self.file = os.path.join(self.directory, name)
self.fields = fields
self.logger = logging.getLogger(name)
self.logger.setLevel(logging.INFO)
self.logger.propagate = False
# Write header
if not resume or not os.path.exists(self.file):
with open(self.file, 'w') as f:
f.write(delimiter.join(fields) + '\n')
file_handler = logging.FileHandler(self.file)
# Adding underscore to avoid clashes with reserved words from logging
field_tmpl = delimiter.join([f'{{_{x}}}' for x in fields])
file_handler.setFormatter(logging.Formatter(field_tmpl, style='{'))
self.logger.addHandler(file_handler)
def validate_paths(self):
stgs = self.store['user_settings']
paths = stgs['base_dirs']
paths = list(set([boardom.process_path(path) for path in paths]))
stgs['base_dirs'] = [
path for path in paths if not is_subdir(path, paths)
]
def __init__(self, directory, db_map_size=10485760, readonly=False):
self.readonly = readonly
directory = bd.process_path(directory, create=True)
self.db_dirname = os.path.join(directory, 'lmdb_data')
self.env = lmdb.open(self.db_dirname,
subdir=True,
readonly=self.readonly)
def __init__(
self,
data_root_path,
only_color=False,
mode='training',
preprocess=None,
load_fn=None,
need_labels=True,
):
super().__init__()
modes = {
'training': 'train_standard',
'validation': 'val',
'testing': 'test',
}
assert mode in modes.keys()
data_root_path = bd.process_path(data_root_path)
flist_root = os.path.join(data_root_path, mode)
txt_file_list = 'places365_{0}{1}.txt'.format(
modes[mode], '_color' if only_color else '')
txt_file_list = os.path.join(data_root_path, txt_file_list)
with open(txt_file_list) as file:
contents = file.readlines()
self.need_labels = need_labels
if mode == 'testing':
flist = [x.strip().split(' ')[0] for x in contents]
self.file_list = [os.path.join(flist_root, x) for x in flist]
self.need_labels = False
if mode == 'validation':
flist, labels = zip(*[x.strip().split(' ') for x in contents])
self.file_list = [os.path.join(flist_root, x) for x in flist]
self.labels = [int(x) for x in labels]
else:
flist, labels = zip(*[x.strip().split(' ') for x in contents])
self.file_list = [os.path.join(flist_root, x[1:]) for x in flist]
self.labels = [int(x) for x in labels]
class_name_list = os.path.join(data_root_path,
'categories_places365.txt')
with open(class_name_list) as file:
contents = file.readlines()
cls_ind = [x.strip().split(' ') for x in contents]
cls_names, indices = zip(*cls_ind)
cls_names = [x[3:] for x in cls_names]
indices = [int(x) for x in indices]
self.cls_to_ind = {c: l for c, l in zip(cls_names, indices)}
self.ind_to_cls = {str(l): c for c, l in zip(cls_names, indices)}
self.preprocess = preprocess
self.load = bd.imread if load_fn is None else load_fn
def __init__(self):
self.directory = bd.process_path(bd.main_file_path())
self.lockfile = os.path.join(self.directory, '.bd.gitlock')
if not os.path.isdir(self.directory):
raise RuntimeError(f'[boardom] Invalid directory {self.directory}')
self.is_git_dir = None
self.status = None
self.status_codes = None
self.status_files = None
self.current_branch = None
self.has_branches = None
self.has_unstaged = None
def setup(self, *cfg_files, extra=None, use_sysargv=True):
if not self._prv['done_setup']:
bd.log('Processing configuration')
arglist = []
# if use_sysargv is bd.Null:
# use_sysargv = self._prv['is_core_config']
if not isinstance(use_sysargv, bool):
raise RuntimeError(
f'use_sysargv expected a bool value. Got {type(use_sysargv)}'
)
if use_sysargv:
arglist += sys.argv[1:]
if cfg_files:
cfg_files = [bd.process_path(f) for f in cfg_files]
arglist += cfg_files
if extra is not None:
arglist += extra
self._update_data_from_parsed(self._parse(arglist))
all_groups = self._prv['all_groups']
if all_groups:
bd.log(f'Groups defined: {self._prv["all_groups"]}')
# Register automatic arguments
self._prv['data']['time_configured'] = _create_datum(
time.strftime("%Y/%m/%d %H:%M:%S")
)
self._prv['data']['process_id'] = _create_datum(_PROCESS_ID)
self._prv['data']['session_path'] = _create_datum(None)
# Leave this here, as the Logger functions called later on (in the subprocess)
# and accesing cfg.project_path and cfg.session_name
# depend on correctly identifying if _prv['done_setup'] is True or False
self._prv['done_setup'] = True
# # If using logger, notify with session_id. This is to change the ID
# # from the execution_id to the session_path
# if bd.BoardomLogger._started:
# # CFG needs to be sent first (lmdb requires session_path)
# bd.BoardomLogger()._send_cfg_full()
# bd.BoardomLogger()._start_lmdb()
bd.log('Config done.')
elif cfg_files:
raise RuntimeError(
'Could not setup from config files as bd.setup() was already called.'
)
return self
def setup_tensorboard_logger_from_cfg(
self,
values_key,
step_key,
mode='scalar',
category='plots',
fields=None,
cfg=None,
**kwargs,
):
cfg = _prepare_cfg(cfg, ['session_path'])
directory = bd.process_path(
os.path.join(cfg.session_path, 'tensorboard'))
return self.setup_tensorboard_logger(values_key, step_key, mode,
category, fields, directory,
**kwargs)
def setup_csv_logger_from_cfg(self,
state_key,
fields,
cfg=None,
delimiter=',',
resume=True):
cfg = _prepare_cfg(cfg, ['session_path'])
directory = bd.process_path(os.path.join(cfg.session_path, 'csv'),
create=True)
return self.setup_csv_logger(
state_key,
fields=fields,
directory=directory,
delimiter=delimiter,
resume=resume,
)
def load_dng(filename, **kwargs):
"""Loads a dng image file from disk into a float32 Numpy Array (OpenCV view).
Requires rawpy.
Args:
filename (str): Name of pfm image file.
**kwargs: Extra keyword arguments to pass to `rawpy.postprocess()`.
"""
import rawpy
filename = bd.process_path(filename)
with rawpy.imread(filename) as raw:
default_kwargs = dict(gamma=(1, 1), no_auto_bright=True, output_bps=16)
default_kwargs.update(kwargs)
img = raw.postprocess(**default_kwargs)
return bd.rgb2bgr(img, -1).astype('float32')
def _parse_cfg_file(self, config_file, subfile=False):
config_file = bd.process_path(config_file)
if not os.path.isfile(config_file):
raise FileNotFoundError(config_file)
with open(config_file) as f:
for count, line in enumerate(f):
yield from self._process_line(line.strip('\n'), count + 1,
config_file)
# Avoid checking opened if it's a subfile
if subfile:
return
for element_type, (_, _, current_list) in self.elements_store.items():
if current_list:
elts = ', '.join(
[f'\'{x}\'' for elm in current_list for x in elm])
raise SyntaxError(
f'Following {element_type}(s) were not closed\n\t{elts}'
f'\nin file: {config_file}')
def imread(filename):
"""Reads an image file from disk into a Numpy Array (OpenCV view).
Args:
filename (str): Name of pfm image file.
"""
filename = bd.process_path(filename)
ext = os.path.splitext(filename)[1]
if ext.lower() == '.pfm':
return load_pfm(filename)
elif ext.lower() == '.dng':
return load_dng(filename)
else:
loaded = cv2.imread(filename,
flags=cv2.IMREAD_ANYDEPTH + cv2.IMREAD_COLOR)
if loaded is None:
raise IOError('Could not read {0}'.format(filename))
else:
return loaded
def setup_tensorboard_logger(
self,
values_key,
step_key,
mode='scalar',
category='plots',
fields=None,
directory='.',
**kwargs,
):
if 'tensorboard' not in self:
directory = bd.process_path(directory, create=True)
writer = SummaryWriter(directory, **kwargs)
self.tensorboard = writer
elif not isinstance(self.tensorboard, SummaryWriter):
raise RuntimeError(
f'self.tensorboard is not a SummaryWriter: {type(self.tensorboard)}'
)
else:
writer = self.tensorboard
category = category.rstrip('/') + '/'
if mode in ['scalar', 'scalars']:
get_value_fn = _value_getter(self, values_key, fields)
def log_fn(values, tag=''):
tag = f'_{tag}' if tag != '' else ''
new_cat = category.rstrip('/') + tag.rstrip('/') + '/'
if step_key not in self:
return
step = self[step_key]
for key, val in values.items():
writer.add_scalar(f'{new_cat}{values_key}.{key}',
val,
global_step=step)
else:
raise RuntimeError(
f'Tensorboard logger mode "{mode}" not supported.')
return GenericLogger(get_value_fn, log_fn, values_key, fields)
def load_pfm(filename):
"""Loads a pfm image file from disk into a Numpy Array (OpenCV view).
Supports HDR and LDR image formats.
Args:
filename (str): Name of pfm image file.
"""
filename = bd.process_path(filename)
with open(filename, "r", encoding="ISO-8859-1") as file:
nc = 3 if file.readline().rstrip() == "PF" else 1
width, height = [int(x) for x in file.readline().rstrip().split()]
shape = (height, width, nc)
img = np.fromfile(
file,
'{0}{1}'.format(
"<" if float(file.readline().rstrip()) < 0 else ">", 'f'),
)
img = np.reshape(img, shape)
return np.flip(np.flip(img, 2), 0).copy()
def __init__(self,
data_root_path,
data_extensions,
load_fn,
preprocess=None):
super().__init__()
data_root_path = bd.process_path(data_root_path)
self.file_list = []
for root, _, fnames in sorted(os.walk(data_root_path)):
for fname in fnames:
if any(fname.lower().endswith(extension)
for extension in data_extensions):
self.file_list.append(os.path.join(root, fname))
if len(self.file_list) == 0:
msg = 'Could not find any files with extensions:\n[{0}]\nin\n{1}'
raise RuntimeError(
msg.format(', '.join(data_extensions), data_root_path))
self.preprocess = preprocess
self.load_fn = load_fn
def setup_csv_logger(self, state_key, fields, directory='.', **kwargs):
directory = bd.process_path(directory, create=True)
if 'csv_writers' not in self:
self.csv_writers = {}
csv_writers = self.csv_writers
def get_writer(tag):
tag = f'_{tag}' if tag != '' else ''
desc = f'{state_key.replace(".","_")}{tag}'
writer = csv_writers.get(desc, None)
if writer is None:
writer = bd.CSVLogger(desc,
fields=fields,
directory=directory,
**kwargs)
csv_writers[desc] = writer
return writer
get_value_fn = _value_getter(self, state_key, fields)
def log_fn(values, tag=''):
get_writer(tag)(values)
return GenericLogger(get_value_fn, log_fn, state_key, fields)
def _create_session(cfg, session_name=None):
cfg.setup()
if not cfg._prv['has_core_config']:
raise RuntimeError('Can not create_session without core configuration')
session = Session()
# Configure session name
if isinstance(session_name, str):
_set(cfg, 'session_name', session_name)
elif isinstance(session_name, Callable):
session_name = session_name(cfg)
_set(cfg, 'session_name', session_name)
elif session_name is None:
session_name = _get(cfg, 'session_name')
else:
raise RuntimeError(
f'Unknown type for session_name parameter: {type(session_name)}')
bd.log(f'Creating {session_name} session.')
project_path = bd.process_path(_get(cfg, 'project_path'), create=True)
bd.log(f'Project path: {project_path}.')
session_path = os.path.join(project_path, session_name)
bd.make_dir(session_path)
boardom_path = bd.make_dir(os.path.join(session_path, '.boardom'))
session_file = os.path.join(boardom_path, BD_FILENAME)
# TODO: Improve Management of Session Files
# -- Maybe use a single file?
# -- Maybe add information
if not os.path.exists(session_file):
with open(session_file, 'w') as f:
f.write('42')
# Maybe create log
create_log = _get(cfg, 'log_stdout')
if create_log:
log_name = f'{session_name}_{_PROCESS_ID}.log'
logdir = os.path.join(session_path, 'log')
logdir = bd.process_path(logdir, create=True)
logfile = os.path.join(logdir, log_name)
logfile = bd.number_file_if_exists(logfile)
bd.log(f'Creating log file at {logfile}')
session.stream_replicator = bd.replicate_std_stream(logfile, 'stdout')
# Maybe copy config files
cfg_files = cfg._prv['cfg_files']
copy_config_files = _get(cfg, 'copy_config_files')
if copy_config_files:
for i, filename in enumerate(cfg_files):
config_path = os.path.join(session_path, 'cfg')
bd.make_dir(config_path)
if i == 0:
bd.log(f'Copying configuration files to {config_path}')
fname, ext = os.path.splitext(filename)
copied_config_filename = f'{fname}_{_PROCESS_ID}{ext}'
bd.copy_file_to_dir(
filename,
config_path,
number=True,
new_name=copied_config_filename,
)
# Maybe save full config
save_full_config = _get(cfg, 'save_full_config')
if save_full_config:
config_path = os.path.join(session_path, 'cfg')
bd.make_dir(config_path)
config_file = os.path.join(config_path, f'full_cfg_{_PROCESS_ID}.bd')
config_file = bd.number_file_if_exists(config_file)
bd.log(f'Saving full configuration at: {config_file}')
# Makes an entry for the saved settings file
def _make_entry(key, val):
if any(isinstance(val, x) for x in [list, tuple]):
val = ' '.join([str(x) for x in val])
return f'{key} {str(val)}'
args_to_print = [
_make_entry(key, val) for key, val in cfg.__dict__.items()
]
args_to_print.sort()
bd.write_string_to_file('\n'.join(args_to_print), config_file)
autocommit = _get(cfg, 'autocommit')
only_run_same_hash = _get(cfg, 'only_run_same_hash')
_, _, autohash = maybe_autocommit(autocommit, only_run_same_hash,
session_path)
pid_fname = f'process.{_PROCESS_ID}'
if autohash is not None:
pid_fname += f'.{autohash}'
# process_dir = bd.make_dir(os.path.join(boardom_path, 'processes'))
# process_id_file = os.path.join(process_dir, pid_fname)
#
# if os.path.exists(process_id_file):
# raise RuntimeError(
# 'Process File Already Exists?!? That is unlucky. Please run again..'
# f'\n id: {process_id_file}'
# )
# else:
# with open(process_id_file, 'w') as f:
# f.write('42')
if _get(cfg, 'print_cfg'):
bd.write('-' * 80)
bd.write(cfg)
bd.write('-' * 80)
cfg._prv['data']['session_path'] = _create_datum(session_path)
return session
def __init__(self, filename, extension):
self.directory = bd.process_path(filename, create=True)
self.fname = os.path.basename(filename)[:-len(extension)]
self.extension = extension
self.current_frame = 1
def _is_valid_config_file(x):
ret = any([x.lower().endswith(y) for y in _CONFIG_EXTENSIONS])
return os.path.isfile(bd.process_path(x)) and ret
def _add_path(self, path):
path = bd.process_path(path)
self.engine.addpath(self.engine.genpath(path))
def _parse(self, arglist):
# First parse all the input files and also get extra arguments
# The data from the config files goes into self._prv['data']
cfg_file_parser = ConfigFileParser()
extra_argv = []
for x in arglist:
if _is_valid_config_file(x):
fname = bd.process_path(x)
self._prv['cfg_files'].append(fname)
for data in cfg_file_parser.parse_cfg_file(fname):
self._add_cfg_file_line_data(data)
else:
extra_argv.append(x)
self._prv['all_groups'] = cfg_file_parser.all_groups
# Check that all arguments provided in the config files are registered for the parser
# If not, give meaningful errors.
# This is automatically handled by the parser, but we do this here to easily track
# the config file error
all_argnames = {
x['flag'][2:]: i for i, x in enumerate(self._prv['argparse_arguments'])
}
for arg_name in self._prv['data']:
if arg_name not in all_argnames:
for arg_data in self._prv['data'][arg_name].values():
meta = arg_data['meta']
file, line, count = meta['file'], meta['line'], meta['count']
raise RuntimeError(
f'Could not find registered argument for \'{arg_name}\' '
f'provided in \'{file}\', line {count}:\n\t{line}'
)
# Create parser and all arguments
parser = argparse.ArgumentParser(allow_abbrev=False, conflict_handler='resolve')
for arg in self._prv['argparse_arguments']:
base_flag_name = arg['flag']
arg_name = arg['flag'][2:]
help_str = arg['help'] if 'help' in arg else None
if arg_name.startswith('_'):
raise ValueError(f'Argument \'{arg_name}\' starts with "_".')
parser.add_argument(
base_flag_name,
help=help_str,
**{k: v for k, v in arg.items() if k not in ['flag', 'help']},
)
# Also add all grouped versions of the argument, defined in the config files
if arg_name in self._prv['data']:
for group in self._prv['data'][arg_name]:
# Ignore default group (already added)
if group.is_default:
continue
flag_name = group.build_full_argname(base_flag_name)
if help_str and not group.is_default:
help_str += f' ({group})'
kwargs = {k: v for k, v in arg.items() if k not in ['flag', 'help']}
parser.add_argument(flag_name, help=help_str, **kwargs)
# Generate argv for parser
arg_list = []
for arg_name, data in self._prv['data'].items():
for group, values in data.items():
name = group.build_full_argname(arg_name)
for x in [f'--{name}'] + values['value']:
arg_list.append(x)
return parser.parse_args(arg_list + extra_argv)
def visualize_parameters(
net,
modules=None,
match_names=None,
param_names=None,
tag='',
save_path='.',
histogram=True,
bins=100,
rename_fn=None,
):
"""Visualizes a network's parameters on an image grid or histogram.
Args:
net (nn.Module): The network whose parameters are to be visualized.
modules (list or tuple, optional): List of class definitions for the
modules where the hook is attached e.g. nn.Conv2d (default None).
match_names (list or tuple, optional): List of strings. If any modules
contain one of the strings then the hook is attached (default None).
param_names (list or tuple, optional): List of strings. If any
parameters of the module contain one of the strings then they are
visualized (default None).
tag (str, optional): String tag to attach to saved images (default None).
save_path (str, optional): Path to save visualisation results
(default '.').
histogram (bool, optional): If True then the visualization is a
histrogram, otherwise it's an image grid.
bins (bool, optional): Number of bins for histogram, if `histogram` is
True (default 100).
Note:
* If modules or match_names are not provided then no parameters will be
visualized.
* If param_names are not provided then no parameters will be visualized.
"""
save_path = bd.process_path(save_path, True)
modules = process_none(modules)
match_names = process_none(match_names)
rename_fn = rename_fn or bd.identity
for module_name, mod in net.named_modules():
name_match = any(
[torch.typename(modules).find(x) >= 0 for x in match_names])
instance_match = any([isinstance(mod, x) for x in modules])
if instance_match or name_match:
params = {x: _get_tensor(getattr(mod, x)) for x in param_names}
for tensor_name, data in params.items():
title = '{0}-{1}-{2}'.format(tag, rename_fn(module_name),
tensor_name)
if data is None:
continue
if histogram:
img = bd.torch2cv(data)
df = pd.DataFrame(img.reshape(img.size))
fig, ax = plt.subplots()
df.hist(bins=bins, ax=ax)
fig.savefig(
os.path.join(save_path, '{0}.png'.format(title)))
plt.close(fig)
else:
if data.dim() > 1:
img = bd.torch2cv(bd.make_grid(data))
to_save = (bd.map_range(img) * 255).astype(int)
cv2.imwrite(
os.path.join(save_path, '{0}.png'.format(title)),
to_save,
)
def get_logging_dir_from_cfg(self, cfg=None):
cfg = _prepare_cfg(cfg, ['session_path'])
return bd.process_path(os.path.join(cfg.session_path, 'csv'),
create=True)
def _hook_generator(
do_input=False,
do_output=True,
tag='',
save_path='.',
replace=True,
histogram=True,
bins=100,
mode='forward',
param_names=None,
rename_fn=None,
ext='.jpg',
):
save_path = bd.process_path(save_path, True)
tensor_names = (['in', 'out'] if mode in ['forward', 'parameters'] else
['grad_in', 'grad_out'])
def get_hook(module_name):
counter = 1
def hook(module, inp=None, out=None):
nonlocal counter, tensor_names
rename_module = rename_fn or bd.identity
added_tag = tag if tag == '' else tag + '_'
if mode == 'parameters':
tensors = {
x: _get_tensor(getattr(module, x))
for x in param_names
}
else:
tensor_coll = [
(tensor_names[0], inp, do_input),
(tensor_names[1], out, do_output),
]
tensors = {}
for name, x, to_do in tensor_coll:
if not to_do:
continue
for d in _get_tensor_dict(x, name):
tensors.update(d)
for tensor_name, data in tensors.items():
if data is None:
continue
title_end = '' if replace else '-{0:06d}'.format(counter)
title_end = title_end + '-hist' if histogram else title_end
title = f'{added_tag}{rename_module(module_name)}_{tensor_name}{title_end}'
if histogram:
img = bd.torch2cv(data)
df = pd.DataFrame(img.reshape(img.size))
fig, ax = plt.subplots()
df.hist(bins=bins, ax=ax)
fig.savefig(os.path.join(save_path, f'{title}{ext}'))
plt.close(fig)
else:
if data.dim() > 1:
img = bd.torch2cv(bd.make_grid(data))
to_save = bd.map_range(img, 0, 255).astype(int)
cv2.imwrite(
os.path.join(save_path,
f'{title}{ext}'.format(title)),
to_save,
)
counter = counter + 1
return hook
return get_hook