def __init__(self, cfg, **kwargs):
super().__init__()
self.stride = get_attr_kwargs(cfg, 'stride', default=1, **kwargs)
self.padding = get_attr_kwargs(cfg, 'padding', default=0, **kwargs)
self.dilation = get_attr_kwargs(cfg, 'dilation', default=1, **kwargs)
self.groups = get_attr_kwargs(cfg, 'groups', default=1, **kwargs)
def __init__(self, cfg, **kwargs):
super(StyleLayer, self).__init__()
cfg = self.update_cfg(cfg)
self.z_dim = get_attr_kwargs(cfg, 'z_dim', **kwargs)
self.n_mlp = get_attr_kwargs(cfg, 'n_mlp', **kwargs)
self.num_features = get_attr_kwargs(cfg, 'num_features', **kwargs)
self.eps = get_attr_kwargs(cfg, 'eps', default=1e-5, **kwargs)
self.momentum = get_attr_kwargs(cfg, 'momentum', default=0.1, **kwargs)
self.conv = nn.Conv2d(self.num_features, self.num_features, 3, 1, 1)
# Prepare gain and bias layers
layers = [PixelNorm()]
for i in range(self.n_mlp):
layers.append(nn.Linear(self.z_dim, self.z_dim))
layers.append(nn.LeakyReLU(0.2))
self.style = nn.Sequential(*layers)
self.gain = nn.Linear(in_features=self.z_dim,
out_features=self.num_features)
self.bias = nn.Linear(in_features=self.z_dim,
out_features=self.num_features)
self.noise = NoiseInjection(self.num_features)
self.lrelu = nn.LeakyReLU(0.2)
pass
def __init__(self, cfg, **kwargs):
super().__init__()
self.in_channels = get_attr_kwargs(cfg, 'in_channels', **kwargs)
self.out_channels = get_attr_kwargs(cfg, 'out_channels', **kwargs)
self.kernel_size = get_attr_kwargs(cfg, 'kernel_size', **kwargs)
self.bias = get_attr_kwargs(cfg, 'bias', **kwargs)
self.cfg_ops = get_attr_kwargs(cfg, 'cfg_ops', **kwargs)
self.num_branch = len(self.cfg_ops)
self.shared_weights = nn.Parameter(
torch.randn(self.out_channels, self.in_channels, self.kernel_size,
self.kernel_size))
nn.init.orthogonal_(self.shared_weights.data)
if self.bias:
self.shared_bias = nn.Parameter(torch.randn(self.out_channels))
else:
self.shared_bias = None
self.branches = nn.ModuleList()
for name, cfg_op in self.cfg_ops.items():
branch = build_d2layer(cfg_op,
in_channels=self.in_channels,
out_channels=self.out_channels)
self.branches.append(branch)
pass
def __init__(self, cfg, **kwargs):
"""
"""
# fmt: off
self.torch_fid_stat = cfg.torch_fid_stat
self.num_inception_images = get_attr_kwargs(cfg,
'num_inception_images',
default=50000,
**kwargs)
self.IS_splits = get_attr_kwargs(cfg,
'IS_splits',
default=10,
**kwargs)
self.calculate_FID_use_torch = get_attr_kwargs(
cfg, 'calculate_FID_use_torch', default=False, **kwargs)
self.no_FID = get_attr_kwargs(cfg, 'no_FID', default=False, **kwargs)
# fmt: on
self.logger = logging.getLogger('tl')
if os.path.isfile(self.torch_fid_stat):
self.logger.info(f"Loading torch_fid_stat : {self.torch_fid_stat}")
self.data_mu = np.load(self.torch_fid_stat)['mu']
self.data_sigma = np.load(self.torch_fid_stat)['sigma']
else:
self.logger.warning(
f"torch_fid_stat does not exist: {self.torch_fid_stat}")
# Load inception_v3 network
self.inception_net = self._load_inception_net()
ws = comm.get_world_size()
self.num_inception_images = self.num_inception_images // ws
pass
def __init__(self, cfg, **kwargs):
self.constant = get_attr_kwargs(cfg, 'constant', **kwargs)
self.sample_shape = get_attr_kwargs(cfg, 'sample_shape', default=None, **kwargs)
if isinstance(self.sample_shape, int):
self.sample_shape = [self.sample_shape, ]
pass
def __init__(self, cfg, **kwargs):
low = get_attr_kwargs(cfg, 'low', default=0, **kwargs)
high = get_attr_kwargs(cfg, 'high', default=1, **kwargs)
self.sample_shape = get_attr_kwargs(cfg, 'sample_shape', default=None, **kwargs)
super(Uniform, self).__init__(low=low, high=high)
pass
def __init__(self, cfg, **kwargs):
super(Conv2dAct, self).__init__()
cfg_act = get_attr_kwargs(cfg, 'cfg_act', **kwargs)
cfg_conv = get_attr_kwargs(cfg, 'cfg_conv', **kwargs)
self.conv = build_d2layer(cfg_conv, **kwargs)
self.act = build_d2layer(cfg_act, **kwargs)
def __init__(self, cfg, **kwargs):
in_features = get_attr_kwargs(cfg, 'in_features', **kwargs)
out_features = get_attr_kwargs(cfg, 'out_features', **kwargs)
bias = get_attr_kwargs(cfg, 'bias', default=True, **kwargs)
super(Linear, self).__init__(in_features=in_features,
out_features=out_features,
bias=bias)
def __init__(self, cfg, **kwargs):
# fmt: off
negative_slope = get_attr_kwargs(cfg, 'negative_slope', default=1e-2, **kwargs)
inplace = get_attr_kwargs(cfg, 'inplace', default=False, **kwargs)
# fmt: on
super(LeakyReLU, self).__init__(negative_slope, inplace)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
self.default_label = get_attr_kwargs(cfg,
'default_label',
default=0,
**kwargs)
self.margin = get_attr_kwargs(cfg, 'margin', default=0., **kwargs)
self.gamma = get_attr_kwargs(cfg, 'gamma', default=1., **kwargs)
# fmt: on
pass
def __init__(self, cfg, **kwargs):
params = kwargs['params']
lr = get_attr_kwargs(cfg, 'lr', default=1e-3, **kwargs)
betas = get_attr_kwargs(cfg, 'betas', default=(0.9, 0.999), **kwargs)
eps = get_attr_kwargs(cfg, 'eps', default=1e-8, **kwargs)
weight_decay = get_attr_kwargs(cfg, 'weight_decay', default=0, **kwargs)
super(RAdamReg, self).__init__(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
loc = get_attr_kwargs(cfg, 'loc', default=0, **kwargs)
scale = get_attr_kwargs(cfg, 'scale', default=1, **kwargs)
self.sample_shape = get_attr_kwargs(cfg, 'sample_shape', default=None, **kwargs)
# fmt: on
super(Normal, self).__init__(loc=loc, scale=scale)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
in_channels = get_attr_kwargs(cfg, 'in_channels', **kwargs)
out_channels = get_attr_kwargs(cfg, 'out_channels', **kwargs)
kernel_size = get_attr_kwargs(cfg, 'kernel_size', **kwargs)
stride = get_attr_kwargs(cfg, 'stride', default=1, **kwargs)
padding = get_attr_kwargs(cfg, 'padding', default=0, **kwargs)
dilation = get_attr_kwargs(cfg, 'dilation', default=1, **kwargs)
groups = get_attr_kwargs(cfg, 'groups', default=1, **kwargs)
bias = get_attr_kwargs(cfg, 'bias', default=True, **kwargs)
padding_mode = get_attr_kwargs(cfg,
'padding_mode',
default='zeros',
**kwargs)
# fmt: on
super(Conv2d, self).__init__(in_channels=in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
bias=bias,
padding_mode=padding_mode)
def __init__(self, cfg, **kwargs):
# fmt: off
root = get_attr_kwargs(cfg, 'root', **kwargs)
train = get_attr_kwargs(cfg, 'train', default=True, **kwargs)
transform = get_attr_kwargs(cfg, 'transform', default=None, **kwargs)
download = get_attr_kwargs(cfg, 'download', default=True, **kwargs)
# fmt: on
super(CIFAR100, self).__init__(root=root, train=train, transform=transform, download=download)
logging.getLogger('tl').info(f"number images of CIFAR100: {len(self)}")
pass
def __init__(self, cfg, **kwargs):
super(PAGANRetrain, self).__init__(cfg=cfg, **kwargs)
self.fixed_arc_file = get_attr_kwargs(cfg.trainer, 'fixed_arc_file',
**kwargs)
self.fixed_epoch = get_attr_kwargs(cfg.trainer, 'fixed_epoch',
**kwargs)
self.classes_arcs = self.get_fixed_arc(self.fixed_arc_file,
self.fixed_epoch)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
in_features = get_attr_kwargs(cfg, 'in_features', **kwargs)
out_features = get_attr_kwargs(cfg, 'out_features', **kwargs)
bias = get_attr_kwargs(cfg, 'bias', default=True, **kwargs)
# fmt: on
super(Linear, self).__init__(in_features, out_features, bias)
pass
def __init__(self, cfg, **kwargs):
super(MixedActCell, self).__init__()
in_channels = get_attr_kwargs(cfg, "in_channels", **kwargs)
out_channels = get_attr_kwargs(cfg, "out_channels", **kwargs)
ksize = get_attr_kwargs(cfg, "ksize", default=3, **kwargs)
up_mode = get_attr_kwargs(cfg, "up_mode", **kwargs)
num_skip_in = get_attr_kwargs(cfg, "num_skip_in", default=0, **kwargs)
short_cut = get_attr_kwargs(cfg, "short_cut", default=False, **kwargs)
norm = get_attr_kwargs(cfg, "norm", default=None, **kwargs)
cfg_mix_layer = get_attr_kwargs(cfg, "cfg_mix_layer", **kwargs)
cfg_ops = get_attr_kwargs(cfg, "cfg_ops", **kwargs)
self.c1 = nn.Conv2d(in_channels,
out_channels,
ksize,
padding=ksize // 2)
self.c2 = nn.Conv2d(out_channels,
out_channels,
ksize,
padding=ksize // 2)
self.act1 = build_d2layer(cfg_mix_layer,
cfg_ops=cfg_ops,
num_parameters=in_channels,
out_channels=in_channels)
self.act2 = build_d2layer(cfg_mix_layer,
cfg_ops=cfg_ops,
num_parameters=out_channels,
out_channels=out_channels)
assert up_mode in UP_MODES
self.up_mode = up_mode
self.norm = norm
if norm:
assert norm in NORMS
if norm == 'bn':
self.n1 = nn.BatchNorm2d(in_channels)
self.n2 = nn.BatchNorm2d(out_channels)
elif norm == 'in':
self.n1 = nn.InstanceNorm2d(in_channels)
self.n2 = nn.InstanceNorm2d(out_channels)
else:
raise NotImplementedError(norm)
# inner shortcut
self.c_sc = None
if short_cut:
self.c_sc = nn.Conv2d(in_channels, out_channels, kernel_size=1)
# cross scale skip
self.skip_in_ops = None
if num_skip_in:
self.skip_in_ops = nn.ModuleList([
nn.Conv2d(out_channels, out_channels, kernel_size=1)
for _ in range(num_skip_in)
])
pass
def __init__(self, cfg, **kwargs):
super().__init__()
cfg_act = get_attr_kwargs(cfg, 'cfg_act', **kwargs)
cfg_conv = get_attr_kwargs(cfg, 'cfg_conv', **kwargs)
cfg_cbn = get_attr_kwargs(cfg, 'cfg_cbn', **kwargs)
self.act = build_d2layer(cfg_act, **kwargs)
self.conv = build_d2layer(cfg_conv, **kwargs)
self.cbn = build_d2layer(cfg_cbn,
num_features=kwargs['in_channels'],
**kwargs)
def __init__(self, cfg, **kwargs):
probs = get_attr_kwargs(cfg, 'probs', default=None, **kwargs)
logits = get_attr_kwargs(cfg, 'logits', default=None, **kwargs)
validate_args = get_attr_kwargs(cfg, 'validate_args', default=None, **kwargs)
self.sample_shape = get_attr_kwargs(cfg, 'sample_shape', default=None, **kwargs)
if isinstance(self.sample_shape, int):
self.sample_shape = [self.sample_shape, ]
super(Categorical, self).__init__(probs=probs, logits=logits, validate_args=validate_args)
pass
def __init__(self, cfg, **kwargs):
super().__init__()
cfg = self.update_cfg(cfg)
self.in_channels = get_attr_kwargs(cfg, 'in_channels', **kwargs)
self.n_nodes = get_attr_kwargs(cfg, 'n_nodes', **kwargs)
self.cfg_mix_layer = get_attr_kwargs(cfg, 'cfg_mix_layer', **kwargs)
self.cfg_ops = get_attr_kwargs(cfg, 'cfg_ops', **kwargs)
self.cell_op_idx = get_attr_kwargs(cfg,
'cell_op_idx',
default=None,
**kwargs)
self.num_edges = self.get_edges(self.n_nodes)
self.cfg_keys = list(self.cfg_ops.keys())
self.out_channels = self.in_channels
assert (self.in_channels) % self.n_nodes == 0
self.internal_c = self.in_channels // self.n_nodes
# self.act = nn.ReLU()
self.in_conv = nn.Conv2d(in_channels=self.in_channels,
out_channels=self.internal_c,
kernel_size=1,
stride=1,
padding=0)
# generate dag
edge_idx = 0
self.dag = nn.ModuleList()
for i in range(self.n_nodes):
self.dag.append(nn.ModuleList())
for j in range(1 + i):
if self.cell_op_idx is not None:
op_key = self.cfg_keys[self.cell_op_idx[edge_idx]]
cfg_ops = EasyDict({op_key: self.cfg_ops[op_key]})
edge_idx += 1
op = build_d2layer(
self.cfg_mix_layer, **{
**kwargs, "in_channels": self.internal_c,
"out_channels": self.internal_c,
"cfg_ops": cfg_ops
})
else:
op = build_d2layer(
self.cfg_mix_layer, **{
**kwargs, "in_channels": self.internal_c,
"out_channels": self.internal_c,
"cfg_ops": self.cfg_ops
})
self.dag[i].append(op)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
n_classes = get_attr_kwargs(cfg, 'n_classes', **kwargs)
validate_args = get_attr_kwargs(cfg, 'validate_args', default=None, **kwargs)
self.sample_shape = get_attr_kwargs(cfg, 'sample_shape', default=None, **kwargs)
# fmt: on
if isinstance(self.sample_shape, int):
self.sample_shape = [self.sample_shape, ]
probs = torch.ones(n_classes) * 1./n_classes
super(CategoricalUniform, self).__init__(probs=probs, logits=None, validate_args=validate_args)
pass
def __init__(self, cfg, **kwargs):
# fmt: off
params = kwargs['params']
lr = get_attr_kwargs(cfg, 'lr', **kwargs)
momentum = get_attr_kwargs(cfg, 'momentum', default=0, **kwargs)
dampening = get_attr_kwargs(cfg, 'dampening', default=0, **kwargs)
weight_decay = get_attr_kwargs(cfg, 'weight_decay', default=0, **kwargs)
nesterov = get_attr_kwargs(cfg, 'nesterov', default=False, **kwargs)
# fmt: on
super(SGD, self).__init__(params, lr=lr, momentum=momentum, dampening=dampening,
weight_decay=weight_decay, nesterov=nesterov)
pass
def __init__(self, cfg, params, **kwargs):
# fmt: off
lr = get_attr_kwargs(cfg, 'lr', default=1e-3, **kwargs)
betas = get_attr_kwargs(cfg, 'betas', default=(0.9, 0.999), **kwargs)
eps = get_attr_kwargs(cfg, 'eps', default=1e-8, **kwargs)
weight_decay = get_attr_kwargs(cfg, 'weight_decay', default=0, **kwargs)
amsgrad = get_attr_kwargs(cfg, 'amsgrad', default=False, **kwargs)
# fmt: on
super(Adam, self).__init__(params, lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,
amsgrad=amsgrad)
pass
def __init__(self, cfg, **kwargs):
super(MixedActLayer, self).__init__()
self.out_channels = get_attr_kwargs(cfg, 'out_channels', **kwargs)
self.cfg_ops = get_attr_kwargs(cfg, 'cfg_ops', **kwargs)
self.num_branch = len(self.cfg_ops)
self.branches = nn.ModuleList()
for name, cfg_op in self.cfg_ops.items():
branch = build_d2layer(cfg_op, **kwargs)
self.branches.append(branch)
pass
def __init__(self, cfg, **kwargs):
self.myargs = kwargs['myargs']
self.D = kwargs['D']
self.G = kwargs['G']
self.D_optim = kwargs['D_optim']
self.G_optim = kwargs['G_optim']
self.b = get_attr_kwargs(cfg, 'b', default=1.0, **kwargs)
self.n_critic = get_attr_kwargs(cfg, 'n_critic', default=5, **kwargs)
self.log_every = getattr(cfg, 'log_every', 50)
self.dummy = getattr(cfg, 'dummy', False)
self.device = torch.device(f'cuda:{comm.get_rank()}')
pass
def __init__(self, cfg, **kwargs):
super(ClsControllerRLAlphaFair, self).__init__()
self.myargs = kwargs['myargs']
self.num_layers = get_attr_kwargs(cfg, 'num_layers', **kwargs)
self.num_branches = get_attr_kwargs(cfg, 'num_branches', **kwargs)
self.num_aggregate = get_attr_kwargs(cfg, 'num_aggregate', **kwargs)
self.entropy_weight = get_attr_kwargs(cfg,
'entropy_weight',
default=0.0001,
**kwargs)
self.bl_dec = get_attr_kwargs(cfg, 'bl_dec', **kwargs)
self.child_grad_bound = get_attr_kwargs(cfg, 'child_grad_bound',
**kwargs)
self.log_every_iter = get_attr_kwargs(cfg,
'log_every_iter',
default=50,
**kwargs)
self.cfg_ops = get_attr_kwargs(cfg, 'cfg_ops', **kwargs)
self.device = torch.device(f'cuda:{comm.get_rank()}')
self.baseline = None
self.logger = logging.getLogger('tl')
self.alpha = nn.ParameterList()
for i in range(self.num_layers):
self.alpha.append(
nn.Parameter(1e-4 * torch.randn(1, self.num_branches)))
pass
def __init__(self, cfg, **kwargs):
super().__init__()
cfg = self.update_cfg(cfg)
# fmt: off
self.in_channels = get_attr_kwargs(cfg, 'in_channels', **kwargs)
self.n_nodes = get_attr_kwargs(cfg, 'n_nodes',
**kwargs) # include input node
self.cfg_mix_layer = get_attr_kwargs(cfg, 'cfg_mix_layer', **kwargs)
self.cfg_ops = get_attr_kwargs(cfg, 'cfg_ops', **kwargs)
self.cell_op_idx = get_attr_kwargs(cfg,
'cell_op_idx',
default=None,
**kwargs)
# fmt: on
self.num_edges = self.get_edges(self.n_nodes)
if self.cell_op_idx is not None:
assert self.num_edges == len(self.cell_op_idx)
self.cfg_keys = list(self.cfg_ops.keys())
self.out_channels = self.in_channels
# generate dag
edge_idx = 0
self.dag = nn.ModuleList()
for i in range(1, self.n_nodes):
self.dag.append(nn.ModuleList())
for j in range(i):
if self.cell_op_idx is not None:
op_key = self.cfg_keys[self.cell_op_idx[edge_idx]]
cfg_ops = EasyDict({op_key: self.cfg_ops[op_key]})
edge_idx += 1
op = build_d2layer_v2(
self.cfg_mix_layer, **{
**kwargs, "in_channels": self.in_channels,
"out_channels": self.out_channels,
"cfg_ops": cfg_ops
})
else:
op = build_d2layer_v2(
self.cfg_mix_layer, **{
**kwargs, "in_channels": self.in_channels,
"out_channels": self.out_channels,
"cfg_ops": self.cfg_ops
})
self.dag[i - 1].append(op)
pass
def __init__(self, cfg, **kwargs):
super(CondInstanceNorm2d, self).__init__()
self.in_features = get_attr_kwargs(cfg, 'in_features', **kwargs)
self.out_features = get_attr_kwargs(cfg, 'out_features', **kwargs)
self.eps = get_attr_kwargs(cfg, 'eps', default=1e-5, **kwargs)
self.momentum = get_attr_kwargs(cfg, 'momentum', default=0.1, **kwargs)
# Prepare gain and bias layers
self.gain = build_d2layer(cfg.cfg_fc,
in_features=self.in_features,
out_features=self.out_features)
self.bias = build_d2layer(cfg.cfg_fc,
in_features=self.in_features,
out_features=self.out_features)
def __init__(self, cfg, **kwargs):
kernel_size = get_attr_kwargs(cfg, 'kernel_size', default=2, **kwargs)
stride = get_attr_kwargs(cfg, 'stride', default=None, **kwargs)
padding = get_attr_kwargs(cfg, 'padding', default=0, **kwargs)
ceil_mode = get_attr_kwargs(cfg, 'ceil_mode', default=False, **kwargs)
count_include_pad = get_attr_kwargs(cfg,
'count_include_pad',
default=True,
**kwargs)
# divisor_override = get_attr_kwargs(cfg, 'divisor_override', default=None, **kwargs)
super().__init__(kernel_size=kernel_size,
stride=stride,
padding=padding,
ceil_mode=ceil_mode,
count_include_pad=count_include_pad)
def __init__(self, cfg, **kwargs):
super().__init__()
cfg = self.update_cfg(cfg)
self.cfg_modconv = get_attr_kwargs(cfg, 'cfg_modconv', **kwargs)
self.add_noise = get_attr_kwargs(cfg,
'add_noise',
default=True,
**kwargs)
self.cfg = cfg
self.activate = nn.ReLU()
self.conv = build_d2layer(self.cfg_modconv, **kwargs)
self.noise = NoiseInjectionV2()