def __init__(self, args: Namespace, *args_, **kwargs):
super().__init__(args, *args_, **kwargs)
# restrictions
assert len(self.objectives) == 1
# bench part
benchmark_set = self._parsed_meta_argument(Register.benchmark_sets,
'cls_benchmark',
args,
index=None)
self.benchmark_set = benchmark_set.from_args(args, index=None)
assert isinstance(self.benchmark_set, MiniNASBenchmark)
self.measure_top = split(
self._parsed_argument('measure_top', self.args), int)
# check if the cell architecture was shared during training
self.num_normal = 1
_, arc_shared = find_in_args(self.args, '.arc_shared')
if not arc_shared:
_, cell_order = find_in_args(self.args, '.cell_order')
self.num_normal = cell_order.count('n')
# correlations
self.correlation_cls = []
for name in split(
self._parsed_argument('measure_correlations', self.args)):
self.correlation_cls.append(Register.correlation_metrics.get(name))
def from_args(cls, args: Namespace, index: int = None) -> AbstractDataSet:
# set class attributes
data_shape, target_shape = cls._parsed_arguments(['data_shape', 'target_shape'], args, index=index)
cls.data_raw_shape = Shape(split(data_shape, int))
cls.label_shape = Shape(split(target_shape, int))
# default generation now
return super().from_args(args, index)
def common_s2_extend_args(cls, args_list: [str]):
"""
allow modifying the arguments list before other classes' arguments are dynamically added
this should be used sparsely, as it is hard to keep track of
"""
print(
'\treading arguments of the supernet training to figure out some things:'
)
# use the arguments of the supernet training task to figure out the network design, criterion, metrics, ...
s1_path = find_in_args_list(
args_list, ['{cls_task}.s1_path',
'%s.s1_path' % cls.__name__])
s1_path = split(s1_path)
if len(s1_path) > 1:
print(
'\t\thave multiple s1_paths, will use the arguments of the first path'
)
arguments_path = get_task_config_path(s1_path[0])
with open(arguments_path) as args_file:
args_in_file = json.load(args_file)
all_meta_args_ = all_meta_args(args_list=None,
args_in_file=args_in_file)
to_ignore = [
'cls_task', 'cls_trainer', 'cls_initializers', 'cls_schedulers'
]
meta_names = [a for a in all_meta_args_ if a not in to_ignore]
cls._add_meta_from_argsfile_to_args(all_args=args_list,
meta_keys=meta_names,
args_in_file=args_in_file,
overwrite=False)
# ensure that there is validation data, set split to 0.5 if not already in .json and disable shuffling
cls_data = find_in_args_list(args_list, ['cls_data'])
valid_split = find_in_args_list(
args_list, ['{cls_data}.valid_split',
'%s.valid_split' % cls_data])
if valid_split is None or float(valid_split) <= 0:
args_list.append('--{cls_data}.valid_split=0.5')
args_list.append('--{cls_data}.valid_shuffle=False')
# set all augmentations to use only the test cases
augmentations = find_in_args_list(args_list, ['cls_augmentations'])
for i, s in enumerate(split(augmentations)):
args_list.append('--%s#%d.force_type=test' % (s, i))
# set fixed method and trainer, disable EMA models
print('\tfixed:')
args_list.append('--cls_trainer=%s' % SimpleTrainer.__name__)
args_list.append('--{cls_trainer}.ema_device=disabled')
print('\t\tusing %s as cls_trainer, disabling EMA weights' %
SimpleTrainer.__name__)
args_list.append('--cls_initializers=""')
args_list.append('--cls_scheduler=""')
print('\t\tusing no cls_scheduler or cls_initializers')
def __init__(self, stem: AbstractModule, heads: nn.ModuleList,
cell_configs: dict, cell_partials: dict, cell_order: Union[str, list],
weight_strategies: Union[dict, str, None] = None, **kwargs_to_save):
"""
:param stem:
:param heads:
:param cell_configs: {name: config} dict of saved cells
:param cell_partials: {name: Callable(cell_index)} to create new (search) cells
:param cell_order: str or [str], will be split to [str]
:param weight_strategies: override default strategies,
{strategy name: [cell indices]}, or name used for all, or None
:param kwargs_to_save:
"""
super().__init__(**kwargs_to_save)
self._add_to_submodules(stem=stem)
self._add_to_submodule_lists(heads=heads)
self._add_to_kwargs_np(cell_configs=cell_configs, cell_partials={})
if isinstance(cell_order, str):
cell_order = split(cell_order)
self._add_to_kwargs(cell_order=cell_order)
self._add_to_print_kwargs(weight_strategies=weight_strategies)
self._cell_partials = cell_partials
self.cells = nn.ModuleList()
self._head_positions = {}
self._update_heads()
def __init__(self, *args, layer_widths: str, act_fun: str, use_bn: bool,
use_bias: bool, **kwargs):
super().__init__(model_name=self.__class__.__name__, *args, **kwargs)
self._layer_widths = split(layer_widths, int)
assert len(self._layer_widths
) > 0, "Must have at least one intermediate layer"
bias = use_bias and not use_bn
# first define the network layers, actually build them when we know the input/output sizes
self.stem = LinearLayer(use_bn=use_bn,
bn_affine=use_bias,
act_fun=act_fun,
act_inplace=False,
order='w_bn_act',
bias=bias)
cells = []
for _ in range(len(self._layer_widths) - 1):
cells.append(
LinearLayer(use_bn=use_bn,
bn_affine=use_bias,
act_fun=act_fun,
act_inplace=False,
order='w_bn_act',
bias=bias))
self.cells = nn.ModuleList(cells)
self.heads = nn.ModuleList([
LinearLayer(use_bn=False,
act_fun='identity',
act_inplace=False,
order='w_bn',
bias=True)
])
def init_multiple(cls, register_dict: RegisterDict, args: args_type,
split_key: str) -> list:
""" creates list of ArgsInterface objects """
splits = split(items(args)[split_key])
return [
register_dict.get(cls_name)(args, i)
for i, cls_name in enumerate(splits)
]
def scheduler_cls(cls,
optimizer=None,
max_epochs=1,
milestones='50, 100',
gamma=0.1):
milestones = split(milestones, cast_fun=int)
return optim.lr_scheduler.MultiStepLR(optimizer=optimizer,
milestones=milestones,
gamma=gamma)
def arg_list_from_json(paths: str) -> [str]:
args = []
for path in split(paths):
path = replace_standard_paths(path)
print('using config file: %s' % path)
with open(path) as config_file:
config = json.load(config_file)
for k, v in config.items():
args.append('--%s=%s' % (k, v))
return args
def __init__(self, hparams: Namespace):
super().__init__(hparams)
self.update_architecture_weights = True
self.train_loader = None
# mask
for idx in split(self._parsed_argument('mask_indices', self.hparams),
cast_fun=int):
self.strategy_manager.mask_index(idx)
LoggerManager().get_logger().info(
"Globally masking arc choices with index %d" % idx)
def _parsed_meta_arguments(cls,
register_dict: RegisterDict,
meta_name: str,
args: args_type,
index=None):
""" get a list of classes back """
try:
names = items(args)[meta_name]
return [register_dict.get(n) for n in split(names)]
except KeyError:
raise KeyError('Value "%s" not in args' % meta_name)
def __init__(self, strategy_name: str, fused: bool, mixed_cls: str, subset: str):
super().__init__()
self.strategy_name = strategy_name
self.fused = fused
self.mixed_cls = mixed_cls
self.subset = split(subset, int)
if self.fused:
assert len(self.subset) == 0, "Can not use a subset of ops if the op is fused!"
else:
assert (len(self.subset) == 0) or (len(self.subset) >= 2),\
"The primitives subset must have at least two operations!"
def get_dataset(data_kwargs: dict) -> AbstractDataSet:
Builder()
# get the data set
parser = argparse.ArgumentParser()
cls_data = Register.data_sets.get(data_kwargs.get('cls_data'))
cls_data.add_arguments(parser, index=None)
for i, cls_aug in enumerate([Register.augmentation_sets.get(cls) for cls in split(data_kwargs.get('cls_augmentations'))]):
cls_aug.add_arguments(parser, index=i)
data_args = parser.parse_args(args=[])
for k, v in data_kwargs.items():
data_args.__setattr__(k, v)
return cls_data.from_args(data_args, index=None)
def _build2(self, s_in: Shape, s_out: Shape) -> ShapeList:
""" build the network """
# find the search config
if not os.path.isfile(self.search_config_path):
self.search_config_path = Builder.find_net_config_path(
self.search_config_path, pattern='search')
# create a temporary search strategy
tmp_s = RandomChoiceStrategy(max_epochs=1, name='__tmp__')
sm = StrategyManager()
assert len(sm.get_strategies_list(
)) == 0, "can not load when there already is a search network"
sm.add_strategy(tmp_s)
sm.set_fixed_strategy_name('__tmp__')
# create a search network
search_net = Register.builder.load_from_config(self.search_config_path)
assert isinstance(search_net, SearchUninasNetwork)
search_net.build(s_in, s_out)
search_net.set_forward_strategy(False)
# set the architecture, get the config
req_gene = ""
if self.gene == 'random':
search_net.forward_strategy()
gene = sm.get_all_finalized_indices(unique=True, flat=True)
self.model_name = "random(%s)" % str(gene)
req_gene = " (%s)" % self.gene
else:
gene = split(self.gene, int)
l0, l1 = len(sm.get_all_finalized_indices(unique=True)), len(gene)
assert l0 == l1, "number of unique choices in the network (%d) must match length of the gene (%d)" % (
l0, l1)
search_net.forward_strategy(fixed_arc=gene)
config = search_net.config(finalize=True)
# clean up
sm.delete_strategy('__tmp__')
del sm
del search_net
# build the actually used finalized network
LoggerManager().get_logger().info(
"Extracting architecture %s%s from the super-network" %
(gene, req_gene))
self.net = Register.builder.from_config(config)
return self.net.build(s_in, s_out)
def _add_meta_from_argsfile_to_args(cls, all_args: [str], meta_keys: [str], args_in_file: dict, overwrite=True):
""" copy all meta arguments in 'meta_keys' and their respective arguments to the 'all_args' list """
already_added = set()
if not overwrite:
for s in all_args:
already_added.add(s.split('=')[0][2:])
for key_meta in meta_keys:
value_meta = args_in_file.get(key_meta)
value_splits = split(sanitize(value_meta))
for key_cls in value_splits:
for k, v in args_in_file.items():
if k in already_added:
continue
if key_meta in k or key_cls in k:
all_args.append('--%s=%s' % (k, v))
already_added.add(k)
if key_meta == k:
print('\t\tusing "%s" as %s, copying arguments' % (v, key_meta))
def __init__(self, args: Namespace, *args_, **kwargs):
AbstractTask.__init__(self, args, *args_, **kwargs)
# benchmarks
self.same_dataset = self._parsed_argument("same_dataset",
args,
index=None)
benchmark_sets = self._parsed_meta_arguments(Register.benchmark_sets,
'cls_benchmarks',
args,
index=None)
self.benchmark_sets = [
bs.from_args(args, index=i) for i, bs in enumerate(benchmark_sets)
]
# correlations
self.correlation_cls = []
for name in split(
self._parsed_argument('measure_correlations', self.args)):
self.correlation_cls.append(Register.correlation_metrics.get(name))
def build_from_args(self,
args_list: [str],
parser: ArgumentParser = None,
raise_problems=True):
"""
recursively parse known arguments and let child nodes parse theirs
:param parser:
:param args_list:
:param raise_problems:
:return:
"""
if parser is None:
parser = ArgumentParser('tmp parser')
self.args_cls.extend_args(args_list=args_list)
meta_args = self.args_cls.add_meta_arguments(parser)
if len(meta_args) > 0:
args, _ = parser.parse_known_args(args_list)
args = self.args_cls.sanitize_args(args)
for meta in meta_args:
self.metas[meta.argument.name] = meta
meta_name = meta.argument.name
cls_names = split(items(args)[meta_name])
self._print('has meta argument:', meta_name, cls_names)
if len(cls_names) == 0 and meta.optional_for_loading:
self._print('missing optional values for', meta_name)
elif raise_problems and not meta.is_allowed_num(
len(cls_names)):
raise ValueError(
'Only %s classed allowed for %s, have %s' %
(meta.allowed_num, meta_name, str(cls_names)))
for cls_name in cls_names:
self.add_child_meta(meta, cls_name)
for child in self.children[meta.argument.name]:
child.build_from_args(args_list,
parser=parser,
raise_problems=raise_problems)
def __init__(self, args: Namespace, index: int, weights: list):
super().__init__(args, index, weights)
self.topk = split(self._parsed_argument('topk', args, self.index), int)
def mask_architecture_space(args: Namespace,
space: ValueSpace) -> ValueSpace:
_, mask = find_in_args(args, ".mask_indices")
for i in split(mask, int):
space.remove_value(i)
return space
def __init__(self, args: Namespace, index=None):
super().__init__(args, index)
self.path, gene, self.strict = self._parsed_arguments(
['path', 'gene', 'strict'], args, index=index)
self.gene = split(gene, int)
def make_from_single_dir(cls, path: str, space_name: str,
arch_index: int) -> MiniResult:
"""
creating a mini result by parsing a training process
"""
# find gene and dataset in the task config
task_configs = find_all_files(path, extension=name_task_config)
assert len(task_configs) == 1
with open(task_configs[0]) as config_file:
config = json.load(config_file)
gene = config.get('{cls_network}.gene')
gene = split(gene, int)
data_set = get_dataset_from_json(task_configs[0])
data_set_name = data_set.__class__.__name__
# find loss and acc in the tensorboard files
average_last = 5
metric_accuracy_train, metric_loss_train = "train/accuracy/1", "train/loss"
metric_accuracy_test, metric_loss_test = "test/accuracy/1", "test/loss"
tb_files = find_tb_files(path)
assert len(tb_files) > 0
events = read_event_files(tb_files)
loss_train = events.get(metric_loss_train, None)
loss_test = events.get(metric_loss_test, None)
assert (loss_train is not None) and (loss_test is not None)
accuracy_train = events.get(metric_accuracy_train, None)
accuracy_test = events.get(metric_accuracy_test, None)
assert (accuracy_train is not None) and (accuracy_test is not None)
# figure out params and flops by building the network
net_config_path = Builder.find_net_config_path(path)
network = get_network(net_config_path, data_set.get_data_shape(),
data_set.get_label_shape())
# figure out latency at some point
pass
# return result
return MiniResult(
arch_index=arch_index,
arch_str="%s(%s)" % (space_name, ", ".join([str(g)
for g in gene])),
arch_tuple=tuple(gene),
params={data_set_name: network.get_num_parameters()},
flops={data_set_name: network.profile_macs()},
latency={data_set_name: -1},
loss={
data_set_name: {
'train':
np.mean([v.value for v in loss_train[-average_last:]]),
'test':
np.mean([v.value for v in loss_test[-average_last:]]),
}
},
acc1={
data_set_name: {
'train':
np.mean([v.value for v in accuracy_train[-average_last:]]),
'test':
np.mean([v.value for v in accuracy_test[-average_last:]]),
}
},
)
