def _log_device_tuning(self, global_step):
if not ivy.exists(self._writer):
raise Exception('torch must be installed in order to use the file writer for tensorboard logging.')
if not ivy.exists(self._dev_manager):
raise Exception('Cannot log device manager tuning if the device manager does not exist.'
'Please set either of the params: tune_device_allocation=True or tune_splitting=True')
# device allocation
# ToDo: log more useful tuning metrics here
if self._multi_dev and self._spec.tune_device_allocation:
self._writer.add_scalar('dev_tuning/device_alloc/tune_count',
self._dev_manager._da_tune_count, global_step)
self._writer.add_scalar('dev_tuning/device_alloc/unit_tune_count',
self._dev_manager._unit_da_tune_count, global_step)
self._writer.add_scalar('dev_tuning/device_alloc/step_time',
self._dev_manager._da_step_time, global_step)
for ds, split in self._dev_manager._dev_strs_da.items():
self._writer.add_scalar('dev_tuning/device_alloc/split_sizes/{}'.format(ds), split, global_step)
# per-device splitting
# ToDo: log more useful tuning metrics here
if self._spec.tune_splitting:
self._writer.add_scalar('dev_tuning/splitting/tune_count',
self._dev_manager._ds_tune_count, global_step)
self._writer.add_scalar('dev_tuning/splitting/step_time',
self._dev_manager._ds_step_time, global_step)
for ds, split in self._dev_manager._dev_strs_ds.items():
self._writer.add_scalar('dev_tuning/splitting/split_factors/{}'.format(ds), split, global_step)
def _dev_manager_execute_with_grads(self, network, batch):
# ToDo: assign this function in constructor rather than performing checks on each training step
dev_manager_exists = ivy.exists(self._dev_manager)
tuned = not dev_manager_exists or self._dev_manager.tuned
if self._compile_network_once_tuned and tuned:
network.compile_on_next_step()
self._compile_network_once_tuned = False
if self._compile_optimizer_once_tuned and tuned:
self._optimizer.compile_on_next_step()
self._compile_optimizer_once_tuned = False
if ivy.exists(self._dev_manager):
if self._multi_dev:
if not isinstance(batch, ivy.MultiDevContainer):
batch = batch.to_multi_dev(self._spec.dev_strs)
return self._dev_manager.map(distributed={"batch": batch.at_devs()},
to_clone={"network_v": network.v})
ret = None
oom = False
while ret is None:
try:
ret = self._split_execute_with_grads(network, self._spec.dev_strs[0], batch, network.v)
except RuntimeError as e:
if oom:
raise Exception('Out of Memory Error raise twice consecutively {}'.format(e))
oom = True
self._dev_manager.tune_step(oom)
return ret
return self._split_execute_with_grads(network, self._spec.dev_strs[0], batch, network.v)
def __init__(self, dataset_spec, batch_size, starting_idx, num_sequences, window_size=1,
num_workers=1, cache_size=0, unused_key_chains=None, custom_init_fn=None,
container_load_mode='dynamic', custom_container_load_fn=None, preshuffle_data=True,
shuffle_buffer_size=0, with_prefetching=True, queue_timeout=None, post_proc_fn=None,
prefetch_to_devs='gpu:0', single_pass=False, array_strs=None, float_strs=None, uint8_strs=None,
custom_img_strs=None, custom_img_fns=None, custom_strs=None, custom_fns=None, array_mode='pickled',
load_gray_as_rgb=True, containers_to_skip=None, **kwargs):
kw = locals_to_kwargs(locals())
unused_key_chains = ivy.default(unused_key_chains, [])
array_strs = ivy.default(array_strs, [])
float_strs = ivy.default(float_strs, [])
uint8_strs = ivy.default(uint8_strs, [])
custom_img_strs = ivy.default(custom_img_strs, [[]])
custom_img_fns = ivy.default(custom_img_fns, [])
custom_strs = ivy.default(custom_strs, [[]])
custom_fns = ivy.default(custom_fns, [])
containers_to_skip = ivy.default(containers_to_skip, [])
prefetch_to_devs = prefetch_to_devs if ivy.gpu_is_available() or isinstance(prefetch_to_devs, list) else False
assert container_load_mode in ['preload', 'dynamic', 'custom']
if container_load_mode == 'custom':
assert ivy.exists(custom_container_load_fn)
else:
assert ivy.exists(dataset_spec.cont_fname_template)
super(SeqDataLoaderSpec, self).__init__(dataset_spec,
batch_size=batch_size,
window_size=window_size,
starting_idx=starting_idx,
num_sequences=num_sequences,
num_workers=num_workers,
cache_size=cache_size,
unused_key_chains=unused_key_chains,
custom_init_fn=custom_init_fn,
container_load_mode=container_load_mode,
custom_container_load_fn=custom_container_load_fn,
preshuffle_data=preshuffle_data,
shuffle_buffer_size=shuffle_buffer_size,
with_prefetching=with_prefetching,
post_proc_fn=post_proc_fn,
prefetch_to_devs=prefetch_to_devs,
single_pass=single_pass,
array_strs=array_strs,
float_strs=float_strs,
uint8_strs=uint8_strs,
custom_img_strs=custom_img_strs,
custom_img_fns=custom_img_fns,
custom_strs=custom_strs,
custom_fns=custom_fns,
array_mode=array_mode,
load_gray_as_rgb=load_gray_as_rgb,
containers_to_skip=containers_to_skip,
**kwargs)
self.queue_timeout = ivy.default(queue_timeout, ivy.queue_timeout()) # conflicts with ivy.Container argument
self._kwargs = kw
def get_json_args(json_spec_path, keys_to_ignore, keychains_to_ignore, keychain_to_show, defaults=False,
store_duplicates=False, current_dir_only=False, spec_names=None):
spec_names = ivy.default(spec_names,
[item.split('.json')[0] for item in os.listdir(json_spec_path) if '.json' in item])
if defaults:
defaults = '.defaults'
else:
defaults = ''
cont = ivy.Container()
if current_dir_only:
for spec_name in spec_names:
fpath = os.path.join(json_spec_path, spec_name + '.json' + defaults)
if os.path.isfile(fpath):
cont[spec_name] = parse_json_to_cont(fpath)
else:
for spec_name in spec_names:
cont[spec_name] = \
json_spec_from_fpath(json_spec_path, spec_name + '.json' + defaults, store_duplicates)
for keychain_to_ignore in keychains_to_ignore:
if keychain_to_ignore in cont:
cont[keychain_to_ignore] = 'not_shown'
cont = cont.set_at_keys(dict(zip(keys_to_ignore, ['not_shown']*len(keys_to_ignore))))
if ivy.exists(keychain_to_show):
cont = cont[keychain_to_show]
return cont
def close(self) -> None:
"""
Close this trainer, and destroy all child objects or processes which may not be garbage collected.
"""
if ivy.exists(self._dev_manager):
self._dev_manager.__del__()
self._spec.data_loader.close()
def step(self):
total_iterations = min(
self._trainer_global_step +
self._train_steps_per_tune_step,
self._trainer_total_iterations)
self._trainer_global_step = self._trainer.train(
self._trainer_global_step, total_iterations)
self.timestep += 1
ret_dict = {
'timestep': self.timestep,
'cost': ivy.to_numpy(self._trainer.moving_average_loss)
}
if self._trainer_global_step >= self._trainer_total_iterations:
if self._save_at_end:
self._trainer._save()
if self._log_at_end and ivy.exists(
self._trainer._training_batch):
self._trainer._log_scalars()
if self._vis_at_end:
dl = self._trainer.spec.data_loader
net = self._trainer.spec.network
tb = self._trainer._training_batch
gs = self._trainer._global_step
self._trainer._write_image_summaries(dl, net, tb, gs)
ret_dict[tune.result.DONE] = True
return ret_dict
def _log_device_utilization(self, global_step):
if not ivy.exists(self._writer):
raise Exception('torch must be installed in order to use the file writer for tensorboard logging.')
self._writer.add_scalar('dev_util/CPU', ivy.dev_util('cpu'), global_step)
for ds in self._spec.dev_strs:
if 'gpu' not in ds:
continue
ds_formatted = ds.replace(':', '_').capitalize()
self._writer.add_scalar('dev_util/{}'.format(ds_formatted), ivy.dev_util(ds), global_step)
def _log_scalars(self):
if ivy.exists(self._writer):
if self._spec.log_time:
self._writer.add_scalar('time between logs', time.perf_counter() - self._start_time, self._global_step)
if self._spec.log_learning_rate:
self._writer.add_scalar('learning rate', self._learning_rate, self._global_step)
self._write_scalar_summaries(self._spec.data_loader, self._network, self._training_batch,
self._global_step)
self._start_time = time.perf_counter()
def _update_seq_info_for_window(self, seq_info):
if not ivy.exists(seq_info):
return
seq_idx = int(seq_info.seq_idx[0])
seq_len = int(seq_info.length[0])
new_len = self._compute_seq_len(seq_idx, seq_len,
self._spec.containers_to_skip)
seq_info = seq_info.copy()
seq_info.length = ivy.ones_like(seq_info.length) * new_len
return seq_info
def restore(self, checkpoint_path):
checkpoint = ivy.Container.from_disk_as_hdf5(checkpoint_path)
loaded_v = checkpoint.network.map(
lambda x, kc: ivy.variable(ivy.to_dev(x, self._net._dev_str)))
if ivy.exists(self._net.v):
# if build_mode is 'on_call', the network variables will not have been built yet
assert (self._net.v.shapes == loaded_v.shapes).all_true(
assert_is_bool=True)
self._net.v = loaded_v
self._optimizer.set_state(
checkpoint.optimizer.map(
lambda x, kc: ivy.to_dev(x, self._net.spec.dev_strs[0])))
def _log_memory(self, global_step):
if not ivy.exists(self._writer):
raise Exception('torch must be installed in order to use the file writer for tensorboard logging.')
self._writer.add_scalar('memory/RAM/global/percent_used', ivy.percent_used_mem_on_dev('cpu'), global_step)
self._writer.add_scalar('memory/RAM/local/percent_used',
ivy.percent_used_mem_on_dev('cpu', process_specific=True), global_step)
for ds in self._spec.dev_strs:
if 'gpu' not in ds:
continue
ds_formatted = ds.replace(':', '_').capitalize()
self._writer.add_scalar('memory/{}/global/percent_used'.format(ds_formatted),
ivy.percent_used_mem_on_dev(ds), global_step)
def _initialize_model(self, checkpoint_path=None):
self._pre_init()
if self._net_spec.build_mode == 'explicit':
self._network.build()
first_batch = self._spec.data_loader.get_first_batch()
if ivy.exists(self._dev_manager):
self._dev_manager.dim_size = first_batch.shape[0]
# for on_call builds
self._compute_cost(self._network, first_batch[0:1], self._spec.dev_strs[0])
# compile
if self._spec.compile_graph:
valid_modes = ['network', 'optimizer', 'all']
assert self._spec.compile_graph in ['network', 'optimizer', 'all'], 'invalid value for compile_graph, ' \
'must be one of {}'.format(valid_modes)
if self._spec.compile_graph in ['network', 'all']:
self._compile_network_once_tuned = True
if self._spec.compile_graph in ['optimizer', 'all']:
self._compile_optimizer_once_tuned = True
if self._spec.save_spec:
self._save_spec_to_disk()
self._save_info_to_disk()
self._init_checkpoint_manager()
if not checkpoint_path:
checkpoint_path = self._chkpt_manager.latest_checkpoint_fpath
if self._spec.ld_chkpt is True and not ivy.exists(checkpoint_path):
raise Exception('Unable to load checkpoint, no checkpoint files found.')
elif self._spec.ld_chkpt in [True, 'try'] and ivy.exists(checkpoint_path):
self._chkpt.restore(checkpoint_path)
logging.info('loaded checkpoints from {}'.format(checkpoint_path))
starting_iteration = int(checkpoint_path.split('-')[-1].split('.')[0])
logging.info('#--------------#\n# MODEL LOADED #\n#--------------#')
self._post_init()
if ivy.exists(self._spec.starting_iteration):
assert starting_iteration == self._spec.starting_iteration
return starting_iteration
else:
logging.info('#-------------#\n# MODEL BUILT #\n#-------------#')
self._global_step = self._spec.starting_iteration
self._post_init()
return ivy.default(self._spec.starting_iteration, 0)
def _float_img_fn(self, filepaths_in_window):
imgs = list()
for filepath in filepaths_in_window:
str_path = bytearray(ivy.to_numpy(filepath).tolist()).decode()
full_path = os.path.abspath(
os.path.join(self._container_data_dir, str_path))
if not ivy.exists(cv2):
raise Exception(
'in order to use _float_img_fn, opencv for python must be installed.'
'To install opencv, run pip install opencv-python.')
img_rgba = cv2.imread(full_path, -1)
img = ivy.array(
np.frombuffer(img_rgba.tobytes(),
np.float32).reshape((1, ) + img_rgba.shape[:-1]))
imgs.append(img)
return ivy.concatenate(imgs, 0)
def build_data_loader(data_loader_class=None,
dataset_dirs_args=None,
dataset_dirs_class=None,
dataset_dirs=None,
dataset_spec_args=None,
dataset_spec_class=None,
dataset_spec=None,
data_loader_spec_args=None,
data_loader_spec_class=None,
data_loader_spec=None,
json_spec_path=None,
spec_cont=None,
class_priority=False):
"""
build data loader
"""
# build data loader specification
data_loader_spec = ivy.default(
data_loader_spec,
build_data_loader_spec(
dataset_dirs_args=dataset_dirs_args,
dataset_dirs_class=dataset_dirs_class,
dataset_dirs=dataset_dirs,
dataset_spec_args=dataset_spec_args,
dataset_spec_class=dataset_spec_class,
dataset_spec=dataset_spec,
data_loader_spec_args=data_loader_spec_args,
data_loader_spec_class=data_loader_spec_class,
json_spec_path=json_spec_path,
spec_cont=spec_cont))
# override data_loader_class if specified in data_loader_spec
data_loader_class = ivy.default(ivy.default(
_import_arg_specified_class_if_present(data_loader_spec, 'data_loader_class'),
data_loader_class, rev=class_priority),
None)
# verify data_loader_class exists
if not ivy.exists(data_loader_class):
raise Exception('data_loader_class must either be specified in this build_data_loader() method,'
'or data_loader_class attribute must be specified in the data_loader_spec instance')
# return data loader
return data_loader_class(data_loader_spec)
def build_network(network_class=None,
dataset_dirs_args=None,
dataset_dirs_class=None,
dataset_dirs=None,
dataset_spec_args=None,
dataset_spec_class=None,
dataset_spec=None,
network_spec_args=None,
network_spec_class=None,
network_spec=None,
json_spec_path=None,
spec_cont=None,
class_priority=False):
"""
build network
"""
# build network specification
network_spec = ivy.default(
network_spec,
build_network_specification(
dataset_dirs_args=dataset_dirs_args,
dataset_dirs_class=dataset_dirs_class,
dataset_dirs=dataset_dirs,
dataset_spec_args=dataset_spec_args,
dataset_spec_class=dataset_spec_class,
dataset_spec=dataset_spec,
network_spec_args=network_spec_args,
network_spec_class=network_spec_class,
json_spec_path=json_spec_path,
spec_cont=spec_cont))
# override network_class if specified in network_spec
network_class = ivy.default(ivy.default(
_import_arg_specified_class_if_present(network_spec, 'network_class'),
network_class, rev=class_priority),
None)
# verify network_class exists
if not ivy.exists(network_class):
raise Exception('network_class must either be specified in this build_network() method,'
'or network_class attribute must be specified in the network_spec instance')
# network
return network_class(network_spec)
def _save_info_to_disk(self):
info_dir = os.path.join(self._spec.log_dir, 'info')
os.makedirs(info_dir, exist_ok=True)
info_filepath = _get_valid_filepath(info_dir, 'info', '.txt')
if not ivy.exists(git):
logging.warning('no gitpython installation found, not saving git commit hash to disk. '
'To install gitpython, run pip install gitpython.')
return
try:
repo = git.Repo(search_parent_directories=True)
sha = repo.head.object.hexsha
except (git.exc.InvalidGitRepositoryError, ValueError):
sha = 'NOT A GIT REPO'
with open(info_filepath, 'w+') as info_file:
info_file.writelines(['time of execution:\n',
str(datetime.now()) + '\n\n',
'git commit hash at time of execution:\n',
sha + '\n'])
def to_filepaths(self):
if not ivy.exists(self._fpath_template):
raise Exception(
'to_filepaths method is not valid if fpath_template has not been specified'
'in the constructor.')
seq_idxs = self._seq_idxs.values()
sizes = [
self._raw_sizes
if self._constant_size else self._raw_sizes[seq_idx]
for seq_idx in seq_idxs
]
rets = [[
self._fpath_template % (seq_idx, win_idx)
for win_idx in range(size) if not SeqDataLoader._skip_cont(
seq_idx, win_idx, self._conts_to_skip)
] for seq_idx, size in zip(seq_idxs, sizes)]
return [
r + [''] * (self._max_seq_len - len(r)) for r in rets
if ''.join(r) != ''
]
def prune_checkpoints_in_dir(chkpts_dir, cutoff, last_only, remove_all):
print('pruning checkpoints in {}'.format(chkpts_dir))
checkpoint_fnames = os.listdir(chkpts_dir)
if len(checkpoint_fnames) == 0:
print('No checkpoints found in {}'.format(chkpts_dir))
return
checkpoint_fnames.sort(key=lambda x: int(x.split('-')[-1].split('.')[0]))
if last_only:
[
os.remove(os.path.join(chkpts_dir, cfn))
for cfn in checkpoint_fnames[:-1]
]
return
if remove_all:
[os.remove(os.path.join(chkpts_dir, cfn)) for cfn in checkpoint_fnames]
return
for checkpoint_fname in checkpoint_fnames:
checkpoint_val = int(checkpoint_fname.split('-')[-1].split('.')[0])
if ivy.exists(cutoff) and checkpoint_val > cutoff:
os.remove(os.path.join(chkpts_dir, checkpoint_fname))
def _custom_img_fn(self, filepaths_in_window, fn):
imgs = list()
for filepath in filepaths_in_window:
str_path = bytearray(ivy.to_numpy(filepath).tolist()).decode()
full_path = os.path.abspath(
os.path.join(self._container_data_dir, str_path))
if not ivy.exists(cv2):
raise Exception(
'in order to use _custom_img_fn, opencv for python must be installed.'
'To install opencv, run pip install opencv-python.')
img_raw = cv2.imread(full_path, -1)
img = fn(img_raw)
imgs.append(img)
img0 = imgs[0]
if isinstance(img0, ivy.Container):
return ivy.Container.concat(imgs, 0)
elif ivy.is_array(img0):
return ivy.concatenate(imgs, 0)
else:
raise Exception(
'custom image functions should either return an array or an ivy.Container instance,'
'but found {} or type {}'.format(img0, type(img0)))
def _uint8_img_fn(self, filepaths_in_window):
imgs = list()
for filepath in filepaths_in_window:
str_path = bytearray(ivy.to_numpy(filepath).tolist()).decode()
full_path = os.path.abspath(
os.path.join(self._container_data_dir, str_path))
if not ivy.exists(cv2):
raise Exception(
'in order to use _uint8_img_fn, opencv for python must be installed.'
'To install opencv, run pip install opencv-python.')
img_rgb = cv2.imread(full_path, -1)
if len(img_rgb.shape) == 2:
if not self._spec.load_gray_as_rgb:
raise Exception(
'Found an image with shape {}, but load_gray_as_rgb is set to False.'
'Set this to True in order to tile grayscale images to RGB.'
.format(img_rgb.shape))
img_rgb = np.tile(np.expand_dims(img_rgb, -1), (1, 1, 3))
img = ivy.array(np.expand_dims(img_rgb.astype(np.float32),
0)) / 255
imgs.append(img)
return ivy.concatenate(imgs, 0)
def _log_nested(self, nest, global_step, name_hierarchy, spec):
if not ivy.exists(self._writer):
raise Exception('torch must be installed in order to use the file writer for tensorboard logging.')
if 'global_vector_norm' in spec:
self._writer.add_scalar(name_hierarchy + '/global vector norm',
ivy.to_scalar(ivy.to_native(nest.vector_norm(global_norm=True))), global_step)
for k, v in nest.items():
new_name_hierarchy = name_hierarchy + '/' + k
if isinstance(v, dict):
self._log_nested(v, global_step, new_name_hierarchy, spec)
else:
if 'mean' in spec:
self._writer.add_scalar(new_name_hierarchy + '/mean',
ivy.to_scalar(ivy.to_native(ivy.reduce_mean(v))), global_step)
if 'abs_mean' in spec:
self._writer.add_scalar(new_name_hierarchy + '/abs mean',
ivy.to_scalar(ivy.to_native(ivy.reduce_mean(ivy.abs(v)))), global_step)
if 'var' in spec:
self._writer.add_scalar(new_name_hierarchy + '/var',
ivy.to_scalar(ivy.to_native(ivy.reduce_var(v))), global_step)
if 'abs_var' in spec:
self._writer.add_scalar(new_name_hierarchy + '/abs var',
ivy.to_scalar(ivy.to_native(ivy.reduce_var(ivy.abs(v)))), global_step)
if 'min' in spec:
self._writer.add_scalar(new_name_hierarchy + '/min',
ivy.to_scalar(ivy.to_native(ivy.reduce_min(v))), global_step)
if 'abs_min' in spec:
self._writer.add_scalar(new_name_hierarchy + '/abs min',
ivy.to_scalar(ivy.to_native(ivy.reduce_min(ivy.abs(v)))), global_step)
if 'max' in spec:
self._writer.add_scalar(new_name_hierarchy + '/max',
ivy.to_scalar(ivy.to_native(ivy.reduce_max(v))), global_step)
if 'abs_max' in spec:
self._writer.add_scalar(new_name_hierarchy + '/abs max',
ivy.to_scalar(ivy.to_native(ivy.reduce_max(ivy.abs(v)))), global_step)
if 'vector_norm' in spec:
self._writer.add_scalar(new_name_hierarchy + '/vector norm',
ivy.to_scalar(ivy.to_native(ivy.vector_norm(v))), global_step)
def print_json_args(base_dir=None, keys_to_ignore=None, keychains_to_ignore=None):
if not ivy.exists(base_dir):
base_dir = os.getcwd()
ivy.set_framework('numpy')
parser = argparse.ArgumentParser()
parser.add_argument('-sd', '--sub_directory', type=str,
help='A sub-directory to print the json args for, default is base_dir passed in.')
parser.add_argument('-dd', '--diff_directory', type=str,
help='The directory from which to compare the difference in specifications.')
parser.add_argument('-kti', '--keys_to_ignore', type=str, default=keys_to_ignore,
help='Keys to ignore when printing the specification.')
parser.add_argument('-kcti', '--keychains_to_ignore', type=str, default=keychains_to_ignore,
help='Key-chains to ignore when printing the specification.')
parser.add_argument('-kcts', '--keychain_to_show', type=str,
help='The key-chain to show. Default is None, in which case all key-chains are shown.')
parser.add_argument('-sn', '--spec_names', type=str,
help='The specification names for the json files. Default is ivy_builder defaults of'
'[ dataset_dirs | dataset | data_loader| network | trainer |]')
parser.add_argument('-d', '--show_defaults', action='store_true',
help='Whether to show the default json arguments.'
'If unset then the current arguments are shown, not the defaut values.')
parser.add_argument('-c', '--current_dir_only', action='store_true',
help='Whether to only show the json arguments for the current directory,'
'without searching through parent directories also.')
parser.add_argument('-sdo', '--show_diff_only', action='store_true',
help='Whether to only show the difference between the current directory'
'and the diff directory.')
parser.add_argument('-sso', '--show_same_only', action='store_true',
help='Whether to only show the same entries between the current directory'
'and the diff directory.')
parsed_args = parser.parse_args()
if (parsed_args.show_diff_only or parsed_args.show_same_only) and not parsed_args.diff_directory:
raise Exception('show_diff_only and show_same_only flags are only applicable if diff_directory is set.')
if parsed_args.show_diff_only and parsed_args.show_same_only:
raise Exception('show_diff_only and show_same_only cannot both be set, please choose one to set.')
if ivy.exists(parsed_args.spec_names):
spec_names = [kc[1:-1] for kc in ''.join(parsed_args.spec_names[1:-1]).split(', ')]
else:
spec_names = None
if ivy.exists(parsed_args.sub_directory):
sub_dir = os.path.normpath(os.path.join(base_dir, parsed_args.sub_directory))
else:
sub_dir = base_dir
if ivy.exists(parsed_args.keys_to_ignore):
keys_to_ignore = [kc[1:-1] for kc in ''.join(parsed_args.keys_to_ignore[1:-1]).split(', ')]
else:
keys_to_ignore = list()
if ivy.exists(parsed_args.keychains_to_ignore):
keychains_to_ignore = [kc[1:-1] for kc in ''.join(parsed_args.keychains_to_ignore[1:-1]).split(',')]
else:
keychains_to_ignore = list()
these_json_args = get_json_args(
sub_dir, keys_to_ignore, keychains_to_ignore, parsed_args.keychain_to_show, parsed_args.show_defaults,
store_duplicates=True, current_dir_only=parsed_args.current_dir_only, spec_names=spec_names)
if ivy.exists(parsed_args.diff_directory):
other_sub_dir = os.path.normpath(os.path.join(base_dir, parsed_args.diff_directory))
if other_sub_dir == sub_dir:
raise Exception('Invalid diff_directory {} selected, it is the same as the sub_directory {}.'.format(
other_sub_dir, sub_dir))
other_json_args = get_json_args(
other_sub_dir, keys_to_ignore, keychains_to_ignore, parsed_args.keychain_to_show, parsed_args.show_defaults,
store_duplicates=True, current_dir_only=parsed_args.current_dir_only, spec_names=spec_names)
diff_keys = 'diff'
for sub_folder, other_sub_folder in zip(sub_dir.split('/'), other_sub_dir.split('/')):
if sub_folder != other_sub_folder:
diff_keys = [sub_folder, other_sub_folder]
break
if parsed_args.show_diff_only:
mode = 'diff_only'
elif parsed_args.show_same_only:
mode = 'same_only'
else:
mode = 'all'
diff_json_args = ivy.Container.diff(these_json_args, other_json_args, mode=mode, diff_keys=diff_keys)
keyword_color_dict = {'duplicated': 'magenta'}
if isinstance(diff_keys, list):
diff_keys_dict = dict(zip(diff_keys, ['red'] * 2))
keyword_color_dict = {**keyword_color_dict, **diff_keys_dict}
print(ivy.Container(diff_json_args, keyword_color_dict=keyword_color_dict))
else:
print(ivy.Container(these_json_args, keyword_color_dict={'duplicated': 'magenta'}))
ivy.unset_framework()
def _get_dataset(self, starting_example, ending_example):
class ContainerIdxMap:
def __init__(self,
sizes,
fpath_template=None,
seq_idxs=None,
start=None,
end=None,
max_seq_len=None,
conts_to_skip=None,
pruned_sizes=None):
if isinstance(sizes, (tuple, list)):
pruned_sizes = ivy.default(pruned_sizes, [
SeqDataLoader._compute_seq_len(i, sl, conts_to_skip)
for i, sl in enumerate(sizes)
])
num_empty = sum([ps == 0 for ps in pruned_sizes])
self._raw_sizes = dict(
zip(range(start, end + 1 + num_empty),
sizes[start:end + 1 + num_empty]))
self._pruned_sizes = dict(
zip(range(start, end + 1 + num_empty),
pruned_sizes[start:end + 1 + num_empty]))
elif isinstance(sizes, (int, dict)):
self._raw_sizes = sizes
self._pruned_sizes = ivy.default(pruned_sizes, sizes)
if isinstance(self._pruned_sizes, int):
pruned_dict = dict()
for seq_idx, win_idx in conts_to_skip:
if seq_idx not in pruned_dict:
pruned_dict[seq_idx] = list()
pruned_dict[seq_idx].append(win_idx)
pruned_dict = {
k: len(set(v))
for k, v in pruned_dict.items()
}
pruned_sizes_dict = {
k: self._pruned_sizes - num_pruned
for k, num_pruned in pruned_dict.items()
}
num_empty = sum(
[size == 0 for size in pruned_sizes_dict.values()])
pruned_sizes = collections.defaultdict(
lambda: self._pruned_sizes, pruned_sizes_dict)
else:
num_empty = sum([ps == 0 for ps in self._pruned_sizes])
else:
raise Exception(
'Invalid type for sizes, expected one of int, dict, tuple or list,'
'but found {} or type {}'.format(sizes, type(sizes)))
self._constant_size = isinstance(self._raw_sizes, int)
if max_seq_len:
self._max_seq_len = max_seq_len
else:
self._max_seq_len = self._pruned_sizes if self._constant_size else max(
self._pruned_sizes.values())
self._fpath_template = fpath_template
self._conts_to_skip = conts_to_skip
if seq_idxs:
self._seq_idxs = seq_idxs
else:
vals = [
v
for i, v in enumerate(range(start, end + 1 +
num_empty))
if pruned_sizes[i] > 0
]
keys = range(0, min(end - start + 1 + num_empty,
len(vals)))
self._seq_idxs = dict(zip(keys, vals))
def __getitem__(self, slice_obj):
if isinstance(slice_obj, slice):
seq_idxs = collections.OrderedDict([
(i, self._seq_idxs[idx]) for i, idx in enumerate(
range(slice_obj.start, slice_obj.stop,
ivy.default(slice_obj.step, 1)))
])
elif isinstance(slice_obj, int):
seq_idxs = collections.OrderedDict(
{0: self._seq_idxs[slice_obj]})
else:
raise Exception(
'Invalid type for slice_obj, expected either slice or int,'
'but found {} of type {}'.format(
slice_obj, type(slice_obj)))
if self._constant_size:
sizes = self._raw_sizes
else:
sizes = collections.OrderedDict({
seq_idx: self._raw_sizes[seq_idx]
for seq_idx in seq_idxs.values()
})
return ContainerIdxMap(sizes,
self._fpath_template,
seq_idxs,
max_seq_len=self._max_seq_len,
conts_to_skip=self._conts_to_skip,
pruned_sizes=self._pruned_sizes)
def __len__(self):
return len(self._seq_idxs)
def shuffle(self):
mapped_idxs = list(self._seq_idxs.values())
np.random.shuffle(mapped_idxs)
self._seq_idxs = collections.OrderedDict(
zip(self._seq_idxs.keys(), mapped_idxs))
def to_idxs(self):
seq_idxs = self._seq_idxs.values()
sizes = [
self._raw_sizes
if self._constant_size else self._raw_sizes[seq_idx]
for seq_idx in seq_idxs
]
rets = [[(seq_idx, win_idx) for win_idx in range(size)
if not SeqDataLoader._skip_cont(
seq_idx, win_idx, self._conts_to_skip)]
for seq_idx, size in zip(seq_idxs, sizes)]
return [
r + [(None, None)] * (self._max_seq_len - len(r))
for r in rets if list(set(r)) != [None]
]
def to_filepaths(self):
if not ivy.exists(self._fpath_template):
raise Exception(
'to_filepaths method is not valid if fpath_template has not been specified'
'in the constructor.')
seq_idxs = self._seq_idxs.values()
sizes = [
self._raw_sizes
if self._constant_size else self._raw_sizes[seq_idx]
for seq_idx in seq_idxs
]
rets = [[
self._fpath_template % (seq_idx, win_idx)
for win_idx in range(size) if not SeqDataLoader._skip_cont(
seq_idx, win_idx, self._conts_to_skip)
] for seq_idx, size in zip(seq_idxs, sizes)]
return [
r + [''] * (self._max_seq_len - len(r)) for r in rets
if ''.join(r) != ''
]
@property
def sizes(self):
return self._pruned_sizes
# container filepaths
if self._spec.container_load_mode in ['preload', 'dynamic']:
fpath_template = os.path.join(
self._container_data_dir,
self._spec.dataset_spec.cont_fname_template)
else:
fpath_template = None
container_idx_map = ContainerIdxMap(
self._spec.dataset_spec.unpruned_sequence_lengths,
fpath_template,
start=starting_example,
end=ending_example,
conts_to_skip=self._spec.containers_to_skip)
if self._spec.num_sequences != -1:
container_idx_map = container_idx_map[0:self._spec.num_sequences]
# shuffle sequences
if self._spec.preshuffle_data:
container_idx_map.shuffle()
# extract sequence lengths
if self._fixed_sequence_length:
self._sequence_lengths =\
collections.OrderedDict(zip(range(len(container_idx_map)),
[self._spec.dataset_spec.sequence_lengths] * len(container_idx_map)))
self._windows_per_seq = self._sequence_lengths[
0] - self._window_size + 1
# windowing values
window_idxs_per_seq = ivy.reshape(
ivy.arange(self._windows_per_seq, 0, 1),
(self._windows_per_seq, 1))
gather_idxs_list = list()
for x in window_idxs_per_seq:
gather_idxs_list.append(
ivy.expand_dims(
ivy.arange(x[0] + self._window_size, x[0], 1), 0))
gather_idxs = ivy.concatenate(gather_idxs_list, 0)
self._gather_idxs = \
ivy.to_numpy(ivy.reshape(gather_idxs, (self._windows_per_seq * self._window_size, 1))).tolist()
else:
self._sequence_lengths = container_idx_map.sizes
# maybe pre-load containers
if self._spec.container_load_mode == 'preload':
# load containers with vector data and image filepath entries
container_slices = self._get_containers_w_filepath_img_entries_as_tensor_slices(
container_idx_map.to_filepaths())
if self._first_frame_validity_fn is not None:
container_slices =\
self._first_frame_validity_fn(container_slices, [ending_example - starting_example + 1])
# prune unwanted chains of keys
if 'unused_key_chains' in self._spec:
container_slices = self._prune_unused_key_chains(
container_slices)
dataset = Dataset(ivy.Container.list_stack([
c[0]
for c in container_slices.unstack(0, container_slices.shape[0])
], 0),
'base',
container_slices.shape[0],
numpy_loading=True,
cache_size=self._base_cache_size,
queue_timeout=self._spec.queue_timeout)
else:
if self._spec.container_load_mode == 'dynamic':
# load containers with filepath entries
dataset = Dataset(ivy.Container({'fpaths': container_idx_map}),
'base',
len(container_idx_map),
trans_fn=lambda cont: cont.map(
lambda x_, kc: x_.to_filepaths()),
elementwise_query_fn=False,
numpy_loading=True,
cache_size=self._base_cache_size,
queue_timeout=self._spec.queue_timeout)
dataset = dataset.map('loaded_json', self._load_json_files,
self._num_workers.loaded_json)
dataset = dataset.map('parsed_json', self._parse_json_strings,
self._num_workers.parsed_json)
else:
dataset = Dataset(ivy.Container({'idx_map':
container_idx_map}),
'base',
len(container_idx_map),
trans_fn=lambda cont: self._spec.
custom_container_load_fn(self, cont),
elementwise_query_fn=False,
numpy_loading=True,
cache_size=self._base_cache_size,
queue_timeout=self._spec.queue_timeout)
if 'unused_key_chains' in self._spec:
dataset = dataset.map('keychain_pruned',
self._prune_unused_key_chains,
self._num_workers.keychain_pruned)
if self._first_frame_validity_fn is not None:
dataset = dataset.map(
'valid_first_frames',
lambda x_: self._first_frame_validity_fn(x_, None),
self._num_workers.valid_first_frames)
if not (self._spec.dataset_spec.sequence_lengths == 1
and self._window_size == 1):
# ToDo: add other conditionals which make the loading more efficient if only one of the
# above two conditions is True
dataset = dataset.map(
'windowed', self._group_container_into_windowed_container,
self._num_workers.windowed)
dataset = dataset.unbatch(
'unbatched',
self._num_workers.unbatched,
batch_sizes=[
max(seq_len, self._window_size) - self._window_size + 1
for seq_len in self._sequence_lengths.values()
if seq_len > 0
])
if self._spec.shuffle_buffer_size > 0:
dataset = dataset.shuffle('shuffled',
self._spec.shuffle_buffer_size,
self._num_workers.shuffled)
dataset = dataset.map('loaded_data',
self._load_data_from_filepath_tensors,
self._num_workers.loaded_data)
dataset = dataset.batch('batched', self._batch_size,
self._num_workers.batched)
dataset = dataset.map(
'from_np',
lambda cont: cont.map(lambda x_, kc: ivy.array(x_, dev_str='cpu')),
self._num_workers.from_np,
numpy_loading=False)
if ivy.exists(self._spec.post_proc_fn):
dataset = dataset.map('post_processed', self._spec.post_proc_fn,
self._num_workers.post_processed)
if self._spec.with_prefetching:
dataset = dataset.prefetch('prefetch')
# ToDo: find way to make pre-fetching to GPU actually pre-fetch, ideally using multi-processing.
# For example, swapping prefetch and to_gpu ops around would work if to_gpu could accept self._num_workers.
if self._spec.prefetch_to_devs:
if isinstance(self._spec.prefetch_to_devs, str):
dataset = dataset.to_dev('to_dev', self._spec.prefetch_to_devs)
elif len(self._spec.prefetch_to_devs) == 1:
dataset = dataset.to_dev('to_dev',
self._spec.prefetch_to_devs[0])
else:
dataset = dataset.to_devs('to_devs',
self._spec.prefetch_to_devs)
return dataset
def build_trainer(data_loader_class=None,
network_class=None,
trainer_class=None,
dataset_dirs_args=None,
dataset_dirs_class=None,
dataset_dirs=None,
dataset_spec_args=None,
dataset_spec_class=None,
dataset_spec=None,
data_loader_spec_args=None,
data_loader_spec_class=None,
data_loader_spec=None,
data_loader=None,
network_spec_args=None,
network_spec_class=None,
network_spec=None,
network=None,
trainer_spec_args=None,
trainer_spec_class=None,
trainer_spec=None,
json_spec_path=None,
spec_cont=None,
class_priority=False):
"""
build trainer
"""
# build trainer spec
trainer_spec = ivy.default(
trainer_spec,
build_trainer_spec(
data_loader_class=data_loader_class,
network_class=network_class,
dataset_dirs_args=dataset_dirs_args,
dataset_dirs_class=dataset_dirs_class,
dataset_dirs=dataset_dirs,
dataset_spec_args=dataset_spec_args,
dataset_spec_class=dataset_spec_class,
dataset_spec=dataset_spec,
data_loader_spec_args=data_loader_spec_args,
data_loader_spec_class=data_loader_spec_class,
data_loader_spec=data_loader_spec,
data_loader=data_loader,
network_spec_args=network_spec_args,
network_spec_class=network_spec_class,
network_spec=network_spec,
network=network,
trainer_spec_args=trainer_spec_args,
trainer_spec_class=trainer_spec_class,
json_spec_path=json_spec_path,
spec_cont=spec_cont))
# override trainer_class if specified in trainer_spec
trainer_class = ivy.default(ivy.default(
_import_arg_specified_class_if_present(trainer_spec, 'trainer_class'),
trainer_class, rev=class_priority),
None)
# verify trainer_class exists
if not ivy.exists(trainer_class):
raise Exception('trainer_class must either be specified in this build_trainer() method,'
'or trainer_class attribute must be specified in the trainer_spec instance')
# return trainer
return trainer_class(trainer_spec)
def custom_container_load_fn(self, cont):
new_cont = ivy.Container()
all_idxs = cont.idx_map.to_idxs()
actions_seqs_list = list()
seq_idxs_seqs_list = list()
idxs_seqs_list = list()
lengths_seqs_list = list()
for seq in all_idxs:
action_arrays_list = list()
seq_idx_arrays_list = list()
idx_arrays_list = list()
found_end = False
j = 0
idx = 0
last_idx = 0
seq_idx = seq[0][0]
for j, (_, idx) in enumerate(seq):
if not ivy.exists(idx) and not found_end:
found_end = True
last_idx = j - 1
if found_end:
idx = last_idx
action_as_list = self._actions_dict[str(seq_idx)][str(idx)]
action_arrays_list.append(
ivy.array(action_as_list, dtype_str='float32')[0])
seq_idx_arrays_list.append(
ivy.array([seq_idx], dtype_str='float32'))
idx_arrays_list.append(ivy.array([idx], dtype_str='float32'))
length_arrays_list = [
ivy.array([last_idx + 1 if found_end else idx + 1],
dtype_str='float32')
] * (j + 1)
action_arrays = ivy.concatenate(action_arrays_list, 0)
actions_seqs_list.append(action_arrays)
seq_idx_arrays = ivy.concatenate(seq_idx_arrays_list, 0)
seq_idxs_seqs_list.append(seq_idx_arrays)
idx_arrays = ivy.concatenate(idx_arrays_list, 0)
idxs_seqs_list.append(idx_arrays)
length_arrays = ivy.concatenate(length_arrays_list, 0)
lengths_seqs_list.append(length_arrays)
new_cont.actions = actions_seqs_list
new_cont.seq_info = ivy.Container()
new_cont.seq_info.seq_idx = seq_idxs_seqs_list
new_cont.seq_info.idx = idxs_seqs_list
new_cont.seq_info.length = lengths_seqs_list
return new_cont
def __init__(
self,
data_loader_class,
network_class,
trainer_class,
dataset_dirs_args: dict = None,
dataset_dirs_class: DatasetDirs.__base__ = DatasetDirs,
dataset_dirs: DatasetDirs = None,
dataset_spec_args: dict = None,
dataset_spec_class: DatasetSpec.__base__ = DatasetSpec,
dataset_spec: DatasetSpec = None,
data_loader_spec_args: dict = None,
data_loader_spec_class: DataLoaderSpec.__base__ = DataLoaderSpec,
data_loader_spec: DataLoaderSpec = None,
data_loader=None,
network_spec_args: dict = None,
network_spec_class: NetworkSpec.__base__ = NetworkSpec,
network_spec: NetworkSpec = None,
network=None,
trainer_spec_args: dict = None,
trainer_spec_class: TrainerSpec.__base__ = TrainerSpec,
trainer_spec: TrainerSpec = None,
trainer=None,
tuner_spec_args: dict = None,
tuner_spec_class: TunerSpec.__base__ = TunerSpec,
tuner_spec: TunerSpec = None,
json_spec_path: str = None,
spec_cont: dict = None):
"""
base class for any tune trainers
"""
if not ivy.exists(tune):
raise Exception(
'ray[tune] is needed in order to use the Tuner class, but it is not installed.'
'Please install via pip install ray[tune]')
self._data_loader_class = data_loader_class
self._network_class = network_class
self._trainer_class = trainer_class
self._dataset_dirs_args = ivy.default(dataset_dirs_args, dict())
self._dataset_dirs_class = dataset_dirs_class
self._dataset_dirs = dataset_dirs
self._dataset_spec_args = ivy.default(dataset_spec_args, dict())
self._dataset_spec_class = dataset_spec_class
self._dataset_spec = dataset_spec
self._data_loader_spec_args = ivy.default(data_loader_spec_args,
dict())
self._data_loader_spec_class = data_loader_spec_class
self._data_loader_spec = data_loader_spec
self._data_loader = data_loader
self._network_spec_args = ivy.default(network_spec_args, dict())
self._network_spec_class = network_spec_class
self._network_spec = network_spec
self._network = network
self._trainer_spec_args = ivy.default(trainer_spec_args, dict())
self._trainer_spec_class = trainer_spec_class
self._trainer_spec = trainer_spec
self._trainer = trainer
self._tuner_spec_args = ivy.default(tuner_spec_args, dict())
self._tuner_spec_class = tuner_spec_class
self._tuner_spec = tuner_spec
self._json_spec_path = json_spec_path
self._spec_cont = spec_cont
# initialized on _setup
self._trainer = None
# builder
while len(ivy.framework_stack) > 0:
logging.info(
'unsetting framework {}, framework stack must be empty when'
'initializing tuner class.'.format(ivy.framework_stack[-1]))
ivy.unset_framework()
self._builder = builder_module
# tuner spec
self._spec = self._builder.build_tuner_spec(
data_loader_class=self._data_loader_class,
network_class=self._network_class,
trainer_class=self._trainer_class,
dataset_dirs_args=self._dataset_dirs_args,
dataset_dirs_class=self._dataset_dirs_class,
dataset_dirs=self._dataset_dirs,
dataset_spec_args=self._dataset_spec_args,
dataset_spec_class=self._dataset_spec_class,
dataset_spec=self._dataset_spec,
data_loader_spec_args=self._data_loader_spec_args,
data_loader_spec_class=self._data_loader_spec_class,
data_loader_spec=self._data_loader_spec,
data_loader=self._data_loader,
network_spec_args=self._network_spec_args,
network_spec_class=self._network_spec_class,
network_spec=self._network_spec,
network=self._network,
trainer_spec_args=self._trainer_spec_args,
trainer_spec_class=self._trainer_spec_class,
trainer_spec=self._trainer_spec,
trainer=self._trainer,
tuner_spec_args=self._tuner_spec_args,
tuner_spec_class=self._tuner_spec_class,
json_spec_path=self._json_spec_path,
spec_cont=self._spec_cont)
self._spec = _convert_tuner_spec(self._spec)
def __init__(self, data_loader_spec: SeqDataLoaderSpec):
super(SeqDataLoader, self).__init__(data_loader_spec)
# cpus
if 'num_workers' in data_loader_spec:
self._total_num_workers = data_loader_spec.num_workers
else:
self._total_num_workers = multiprocessing.cpu_count()
# first frame validity
if 'first_frame_validity_fn' in data_loader_spec:
self._first_frame_validity_fn = data_loader_spec.first_frame_validity_fn
else:
self._first_frame_validity_fn = None
# data loader specification
self._spec = data_loader_spec
self._container_data_dir = os.path.join(
self._spec.dataset_spec.dirs.dataset_dir, 'containers/')
self._batch_size = self._spec.batch_size
self._base_cache_size = self._spec.cache_size * self._spec.batch_size * self._spec.window_size
self._window_size = self._spec.window_size
start_idx = self._spec.starting_idx
end_idx = start_idx + self._spec.num_sequences - 1
# specs before pruning via containers_to_skip
self._spec.dataset_spec.unpruned_sequence_lengths = self._spec.dataset_spec.sequence_lengths
self._spec.unpruned_num_sequences = self._spec.num_sequences
# sequence lengths and windows per sequence
if 'sequence_lengths' in self._spec.dataset_spec:
self._fixed_sequence_length = isinstance(
self._spec.dataset_spec.sequence_lengths, int)
if self._fixed_sequence_length:
self._windows_per_seq = self._spec.dataset_spec.sequence_lengths - (
self._window_size - 1)
else:
# update sequences lengths
self._spec.dataset_spec.sequence_lengths =\
[self._compute_seq_len(i, sl, self._spec.containers_to_skip)
for i, sl in enumerate(self._spec.dataset_spec.sequence_lengths)]
self._spec.num_sequences =\
sum([sl > 0 for sl in self._spec.dataset_spec.sequence_lengths[start_idx:end_idx+1]])
self._windows_per_seq = ivy.array(
self._spec.dataset_spec.sequence_lengths) - (
self._window_size - 1)
else:
self._fixed_sequence_length = False
# new end idx following containers_to_skip pruning
end_idx = start_idx + self._spec.num_sequences - 1
# compute num workers for each component
self._compute_num_workers()
# custom init
self._custom_init_fn = self._spec.custom_init_fn
if ivy.exists(self._custom_init_fn):
self._custom_init_fn(self)
# dataset
self._dataset = self._get_dataset(start_idx, end_idx)
self._iterator = iter(self._dataset)
# dummy batch
self._first_batch = None
# counter
self._counter = 0
