def __init__(self, params, load_path=None, dist_model=False): # set dist_model to False in demo
self.model = backbone.__dict__[params['backbone_arch']](**params['backbone_param'])
# Random init here, so do not load pretrained model when constructing backbones.
utils.init_weights(self.model, init_type='xavier')
# load pretrain here.
if load_path is not None:
utils.load_weights(load_path, self.model)
self.model.cuda()
if dist_model:
self.model = utils.DistModule(self.model)
self.world_size = dist.get_world_size()
else:
self.model = backbone.FixModule(self.model)
self.world_size = 1
if params['optim'] == 'SGD':
self.optim = torch.optim.SGD(
self.model.parameters(), lr=params['lr'],
momentum=0.9, weight_decay=params['weight_decay'])
elif params['optim'] == 'Adam':
self.optim = torch.optim.Adam(
self.model.parameters(), lr=params['lr'],
betas=(params['beta1'], 0.999))
else:
raise Exception("No such optimizer: {}".format(params['optim']))
cudnn.benchmark = True
def init(self):
# initialze the weigths for the LSTM RNN
self.Wi, self.Wf, self.Wo, self.Wc = [], [], [], []
self.Whi, self.Whf, self.Who, self.Whc = [], [], [], []
self.bi, self.bf, self.bo, self.bc = [], [], [], []
for i in range(len(self.rnn_layers)):
n_inputs = self.vocab_size if i == 0 else self.rnn_layers[i-1]
self.Wi.append(init_weights((n_inputs, self.rnn_layers[i]), rng=self.numpy_rng, name='Wi_{}'.format(i)))
self.Wf.append(init_weights((n_inputs, self.rnn_layers[i]), rng=self.numpy_rng, name='Wf_{}'.format(i)))
self.Wo.append(init_weights((n_inputs, self.rnn_layers[i]), rng=self.numpy_rng, name='Wo_{}'.format(i)))
self.Wc.append(init_weights((n_inputs, self.rnn_layers[i]), rng=self.numpy_rng, name='Wc_{}'.format(i)))
self.Whi.append(init_ortho(self.rnn_layers[i], rng=self.numpy_rng, name='Whi_{}'.format(i)))
self.Whf.append(init_ortho(self.rnn_layers[i], rng=self.numpy_rng, name='Whf_{}'.format(i)))
self.Who.append(init_ortho(self.rnn_layers[i], rng=self.numpy_rng, name='Who_{}'.format(i)))
self.Whc.append(init_ortho(self.rnn_layers[i], rng=self.numpy_rng, name='Whc_{}'.format(i)))
self.bi.append(theano.shared(value=np.zeros((self.rnn_layers[i],), dtype=theano.config.floatX), borrow=True, name='bi_{}'.format(i)))
self.bf.append(theano.shared(value=np.zeros((self.rnn_layers[i],), dtype=theano.config.floatX), borrow=True, name='bf_{}'.format(i)))
self.bo.append(theano.shared(value=np.zeros((self.rnn_layers[i],), dtype=theano.config.floatX), borrow=True, name='bo_{}'.format(i)))
self.bc.append(theano.shared(value=np.zeros((self.rnn_layers[i],), dtype=theano.config.floatX), borrow=True, name='bc_{}'.format(i)))
self.Why = init_weights((self.rnn_layers[i], self.vocab_size), rng=self.numpy_rng, name='Why') # hidden to output weights
self.by = theano.shared(value=np.zeros((self.vocab_size,), dtype=theano.config.floatX), borrow=True, name='by') # hidden to output bias
params = [self.Why, self.by]
params += self.Wi + self.Wf + self.Wo + self.Wc
params += self.bi + self.bf + self.bo + self.bc
params += self.Whi + self.Whf + self.Who + self.Whc
return params
def __init__(self, params, dist_model=False):
model_params = params['module']
self.model = models.modules.__dict__[params['module']['arch']](
model_params)
utils.init_weights(self.model, init_type='xavier')
self.model.cuda()
if dist_model:
self.model = utils.DistModule(self.model)
self.world_size = dist.get_world_size()
else:
self.model = models.modules.FixModule(self.model)
self.world_size = 1
if params['optim'] == 'SGD':
self.optim = torch.optim.SGD(self.model.parameters(),
lr=params['lr'],
momentum=0.9,
weight_decay=0.0001)
elif params['optim'] == 'Adam':
self.optim = torch.optim.Adam(self.model.parameters(),
lr=params['lr'],
betas=(params['beta1'], 0.999))
else:
raise Exception("No such optimizer: {}".format(params['optim']))
cudnn.benchmark = True
def __init__(self, z_dim=100, image_channels=1, image_size=28):
super(Generator, self).__init__()
self.z_dim = z_dim
self.image_channels = image_channels
self.image_size = image_size
self.fc = nn.Sequential(
# input is Z, going into a fully connected layer
nn.Linear(self.z_dim, 1024), # (-1, 1024)
nn.BatchNorm1d(1024),
nn.ReLU(),
nn.Linear(1024, 128 * (self.image_size // 4) *
(self.image_size // 4)), # (-1, 128*7*7)
nn.BatchNorm1d(128 * (self.image_size // 4) *
(self.image_size // 4)),
nn.ReLU())
# reshape -> (-1, 128, 7, 7)
self.deconv = nn.Sequential(
nn.ConvTranspose2d(128, 64, 4, 2, 1), # (-1, 64, 14, 14)
nn.BatchNorm2d(64),
nn.ReLU(),
nn.ConvTranspose2d(64, self.image_channels, 4, 2, 1),
# (-1, 3, 28, 28)
nn.Tanh())
init_weights(self)
self._print_info()
def init(self):
# initialze the weigths for the LSTM RNN
# group Wi, Wf, Wo and Wc into a W matrix with 4 times the original number of columns
# group bi, bf, bo and bc into a b matrix with 4 times the original number of columns
# group Whi, Whf, Who and Whc into a Wh matrix with 4 times the original number of columns
# this allows a better usage of the vector units in the CPU/GPU
self.W, self.Wh, self.b = [], [], []
for i in range(len(self.rnn_layers)):
n_inputs = self.vocab_size if i == 0 else self.rnn_layers[i - 1]
self.W.append(
init_weights((n_inputs, 4 * self.rnn_layers[i]),
rng=self.numpy_rng,
name='W_{}'.format(i)))
self.Wh.append(
init_ortho(self.rnn_layers[i],
num_matrices=4,
rng=self.numpy_rng,
name='Wh_{}'.format(i)))
self.b.append(
theano.shared(value=np.zeros((4 * self.rnn_layers[i], ),
dtype=theano.config.floatX),
borrow=True,
name='b_{}'.format(i)))
self.Why = init_weights((self.rnn_layers[i], self.vocab_size),
rng=self.numpy_rng,
name='Why') # hidden to output weights
self.by = theano.shared(value=np.zeros((self.vocab_size, ),
dtype=theano.config.floatX),
borrow=True,
name='by') # hidden to output bias
params = self.W + self.Wh + self.b + [self.Why, self.by]
return params
def build_model(self):
# Weights for fully connected layers
self.w_alice = init_weights("alice_w", [2 * self.N, 2 * self.N])
self.w_bob = init_weights("bob_w", [2 * self.N, 2 * self.N])
self.w_eve1 = init_weights("eve_w1", [self.N, 2 * self.N])
self.w_eve2 = init_weights("eve_w2", [2 * self.N, 2 * self.N])
# Placeholder variables for Message and Key
self.msg = tf.placeholder("float", [None, self.msg_len])
self.key = tf.placeholder("float", [None, self.key_len])
# Alice's network
self.alice_input = tf.concat(concat_dim=1, values=[self.msg, self.key])
self.alice_hidden = tf.nn.sigmoid(
tf.matmul(self.alice_input, self.w_alice))
self.alice_hidden = tf.expand_dims(self.alice_hidden, 2)
self.alice_output = tf.squeeze(conv_layer(self.alice_hidden, "alice"))
# Bob's network
self.bob_input = tf.concat(concat_dim=1,
values=[self.alice_output, self.key])
self.bob_hidden = tf.nn.sigmoid(tf.matmul(self.bob_input, self.w_bob))
self.bob_hidden = tf.expand_dims(self.bob_hidden, 2)
self.bob_output = tf.squeeze(conv_layer(self.bob_hidden, "bob"))
# Eve's network
self.eve_input = self.alice_output
self.eve_hidden1 = tf.nn.sigmoid(tf.matmul(self.eve_input,
self.w_eve1))
self.eve_hidden2 = tf.nn.sigmoid(
tf.matmul(self.eve_hidden1, self.w_eve2))
self.eve_hidden2 = tf.expand_dims(self.eve_hidden2, 2)
self.eve_output = tf.squeeze(conv_layer(self.eve_hidden2, "eve"))
def __init__(self, coding_size, img_channels=3, hidden_dims=(16, 32, 16, 8, 2)):
super(AutoEncoder, self).__init__()
# encoder
self.MaxPool = nn.MaxPool2d(kernel_size=2, stride=2)
self.Conv1 = BasicConvBlock(ch_in=img_channels, ch_out=hidden_dims[0])
self.Conv2 = BasicConvBlock(ch_in=hidden_dims[0], ch_out=hidden_dims[1])
self.Conv3 = BasicConvBlock(ch_in=hidden_dims[1], ch_out=hidden_dims[2])
self.Conv4 = BasicConvBlock(ch_in=hidden_dims[2], ch_out=hidden_dims[3])
self.Conv5 = BasicConvBlock(ch_in=hidden_dims[3], ch_out=hidden_dims[4])
# decoder
self.Up5 = UpConvBlock(ch_in=hidden_dims[4], ch_out=hidden_dims[3])
self.Up4 = UpConvBlock(ch_in=hidden_dims[3], ch_out=hidden_dims[2])
self.Up3 = UpConvBlock(ch_in=hidden_dims[2], ch_out=hidden_dims[1])
self.Up2 = UpConvBlock(ch_in=hidden_dims[1], ch_out=hidden_dims[0])
self.Up1 = nn.Conv2d(hidden_dims[0], img_channels, kernel_size=1, stride=1, padding=0)
self.encoder = nn.Sequential(*[self.Conv1, self.MaxPool, self.Conv2, self.MaxPool,
self.Conv3, self.MaxPool, self.Conv4, self.MaxPool, self.Conv5, nn.Flatten(),
nn.Linear(392, coding_size)])
self.decoder = nn.Sequential(
*[nn.Linear(coding_size, 392), nn.Unflatten(1, (2, 14, 14)), self.Up5, self.Up4, self.Up3, self.Up2,
self.Up1])
init_weights(self.encoder)
init_weights(self.decoder)
def __init__(self, in_channels=1, n_classes=2, feature_scale=2, is_deconv=True, is_batchnorm=True):
super(UNet, self).__init__()
self.in_channels = in_channels
self.feature_scale = feature_scale
self.is_deconv = is_deconv
self.is_batchnorm = is_batchnorm
filters = [64, 128, 256, 512, 1024]
filters = [int(x / self.feature_scale) for x in filters]
# downsampling
self.maxpool = nn.MaxPool2d(kernel_size=2)
self.conv1 = unetConv2(self.in_channels, filters[0], self.is_batchnorm)
self.conv2 = unetConv2(filters[0], filters[1], self.is_batchnorm)
self.conv3 = unetConv2(filters[1], filters[2], self.is_batchnorm)
self.conv4 = unetConv2(filters[2], filters[3], self.is_batchnorm)
self.center = unetConv2(filters[3], filters[4], self.is_batchnorm)
# upsampling
self.up_concat4 = unetUp(filters[4], filters[3], self.is_deconv)
self.up_concat3 = unetUp(filters[3], filters[2], self.is_deconv)
self.up_concat2 = unetUp(filters[2], filters[1], self.is_deconv)
self.up_concat1 = unetUp(filters[1], filters[0], self.is_deconv)
# final conv (without any concat)
self.final = nn.Conv2d(filters[0], n_classes, 1)
# initialise weights
for m in self.modules():
if isinstance(m, nn.Conv2d):
init_weights(m, init_type='kaiming')
elif isinstance(m, nn.BatchNorm2d):
init_weights(m, init_type='kaiming')
def __init__(self, in_size, out_size, is_batchnorm, n=2, ks=3, stride=1, padding=1):
super(unetConv2, self).__init__()
self.n = n
self.ks = ks
self.stride = stride
self.padding = padding
s = stride
p = padding
if is_batchnorm:
for i in range(1, n+1):
conv = nn.Sequential(nn.Conv2d(in_size, out_size, ks, s, p),
nn.BatchNorm2d(out_size),
nn.ReLU(inplace=True),)
setattr(self, 'conv%d'%i, conv)
in_size = out_size
else:
for i in range(1, n+1):
conv = nn.Sequential(nn.Conv2d(in_size, out_size, ks, s, p),
nn.ReLU(inplace=True),)
setattr(self, 'conv%d'%i, conv)
in_size = out_size
# initialise the blocks
for m in self.children():
init_weights(m, init_type='kaiming')
def cross_val(p) -> object:
data_ = load_data(root_dir='./data/', mode='train')
data_, target_, features, date = preprocess_data(data_,
nn=True,
action='multi')
input_size = data_.shape[-1]
output_size = target_.shape[-1]
gts = PurgedGroupTimeSeriesSplit(n_splits=5, group_gap=5)
models = []
tb_logger = pl_loggers.TensorBoardLogger('logs/multiclass_')
for i, (train_idx, val_idx) in enumerate(gts.split(data_, groups=date)):
idx = np.concatenate([train_idx, val_idx])
data = copy.deepcopy(data_[idx])
target = copy.deepcopy(target_[idx])
checkpoint_callback = pl.callbacks.ModelCheckpoint(os.path.join(
'models/', 'multi_class_fold_{}'.format(i)),
monitor='val_auc',
save_top_k=1,
period=10,
mode='max')
model = ResNet(input_size, output_size, p)
if p['activation'] == nn.ReLU:
model.apply(lambda m: init_weights(m, 'relu'))
elif p['activation'] == nn.LeakyReLU:
model.apply(lambda m: init_weights(m, 'leaky_relu'))
train_idx = [i for i in range(0, max(train_idx) + 1)]
val_idx = [i for i in range(len(train_idx), len(idx))]
data[train_idx] = calc_data_mean(data[train_idx],
'./cache',
train=True,
mode='mean')
data[val_idx] = calc_data_mean(data[val_idx],
'./cache',
train=False,
mode='mean')
dataset = FinData(data=data, target=target, date=date, multi=True)
dataloaders = create_dataloaders(dataset,
indexes={
'train': train_idx,
'val': val_idx
},
batch_size=p['batch_size'])
es = EarlyStopping(monitor='val_auc',
patience=10,
min_delta=0.0005,
mode='max')
trainer = pl.Trainer(logger=tb_logger,
max_epochs=100,
gpus=1,
callbacks=[checkpoint_callback, es],
precision=16)
trainer.fit(model,
train_dataloader=dataloaders['train'],
val_dataloaders=dataloaders['val'])
torch.save(model.state_dict(), f'models/fold_{i}_state_dict.pth')
models.append(model)
return models, features
def __init__(self, input_layer_size, hidden_layer_size, output_layer_size):
self.input_layer_size = input_layer_size
self.hidden_layer_size = hidden_layer_size
self.output_layer_size = output_layer_size
self.weight_first = init_weights(hidden_layer_size, input_layer_size)
self.weight_second = init_weights(output_layer_size, hidden_layer_size)
self.learning_rate = 1
self.batch_size = 1
def __init__(self,
feature_dim=784,
latent_dim=10,
input_size=(1, 28, 28),
verbose=False):
super(RES_NET_Classifier, self).__init__()
self.feature_dim = feature_dim
self.latent_dim = latent_dim
self.input_size = input_size
self.cshape = (128, 1, 1)
self.iels = int(np.prod(self.cshape))
self.lshape = (self.iels, )
self.vervose = verbose
self.model = nn.Sequential(
nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(True),
nn.MaxPool2d(2, 2),
)
res_block = []
for i in range(3):
res_block += [ResNet_Block(64, 64)]
res_block += [
nn.Conv2d(64, 128, kernel_size=3, padding=1, stride=2),
nn.BatchNorm2d(128),
nn.ReLU(True),
nn.Conv2d(128, 128, kernel_size=3, padding=1, stride=1),
nn.BatchNorm2d(128),
nn.ReLU(True),
]
for i in range(3):
res_block += [ResNet_Block(128, 128)]
self.model = nn.Sequential(
self.model,
*res_block,
nn.AvgPool2d(2, 2),
Reshape(self.lshape),
# nn.Linear(self.iels, 1024),
# nn.BatchNorm1d(1024),
# nn.ReLU(True),
nn.Linear(self.iels, self.latent_dim),
)
init_weights(self)
if self.vervose:
print(self.model)
def __init__(self,
in_channels=3,
n_classes=2,
feature_scale=2,
is_deconv=True,
is_batchnorm=True,
is_ds=True):
super(UNet_Nested, self).__init__()
self.in_channels = in_channels
self.feature_scale = feature_scale
self.is_deconv = is_deconv
self.is_batchnorm = is_batchnorm
self.is_ds = is_ds
filters = [64, 128, 256, 512, 1024]
filters = [int(x / self.feature_scale) for x in filters]
# downsampling
self.maxpool = nn.MaxPool2d(kernel_size=2)
self.conv00 = unetConv2(self.in_channels, filters[0],
self.is_batchnorm)
self.conv10 = unetConv2(filters[0], filters[1], self.is_batchnorm)
self.conv20 = unetConv2(filters[1], filters[2], self.is_batchnorm)
self.conv30 = unetConv2(filters[2], filters[3], self.is_batchnorm)
self.conv40 = unetConv2(filters[3], filters[4], self.is_batchnorm)
# upsampling
self.up_concat01 = unetUp(filters[1], filters[0], self.is_deconv)
self.up_concat11 = unetUp(filters[2], filters[1], self.is_deconv)
self.up_concat21 = unetUp(filters[3], filters[2], self.is_deconv)
self.up_concat31 = unetUp(filters[4], filters[3], self.is_deconv)
self.up_concat02 = unetUp(filters[1], filters[0], self.is_deconv, 3)
self.up_concat12 = unetUp(filters[2], filters[1], self.is_deconv, 3)
self.up_concat22 = unetUp(filters[3], filters[2], self.is_deconv, 3)
self.up_concat03 = unetUp(filters[1], filters[0], self.is_deconv, 4)
self.up_concat13 = unetUp(filters[2], filters[1], self.is_deconv, 4)
self.up_concat04 = unetUp(filters[1], filters[0], self.is_deconv, 5)
# final conv (without any concat)
self.final_1 = nn.Conv2d(filters[0], n_classes, 1)
self.final_2 = nn.Conv2d(filters[0], n_classes, 1)
self.final_3 = nn.Conv2d(filters[0], n_classes, 1)
self.final_4 = nn.Conv2d(filters[0], n_classes, 1)
# initialise weights
for m in self.modules():
if isinstance(m, nn.Conv2d):
init_weights(m, init_type='kaiming')
elif isinstance(m, nn.BatchNorm2d):
init_weights(m, init_type='kaiming')
def __init__(self, params, dist_model=False):
netD_params = params['discriminator']
# define model
self.model = backbone.__dict__[params['backbone_arch']](
**params['backbone_param'])
utils.init_weights(self.model, init_type='xavier')
self.model.cuda()
if dist_model:
self.model = utils.DistModule(self.model)
self.world_size = dist.get_world_size()
else:
self.model = backbone.FixModule(self.model)
self.world_size = 1
if params['optim'] == 'SGD':
self.optim = torch.optim.SGD(self.model.parameters(),
lr=params['lr'],
momentum=0.9,
weight_decay=params['weight_decay'])
elif params['optim'] == 'Adam':
self.optim = torch.optim.Adam(self.model.parameters(),
lr=params['lr'],
betas=(params['beta1'], 0.999))
else:
raise Exception("No such optimizer: {}".format(params['optim']))
# define netD
self.netD = backbone.__dict__[netD_params['arch']](
**netD_params['arch_param'])
self.netD.cuda()
if dist_model:
self.netD = utils.DistModule(self.netD)
else:
self.netD = backbone.FixModule(self.netD)
if netD_params['optim'] == 'SGD':
self.optimD = torch.optim.SGD(self.netD.parameters(),
lr=netD_params['lr'],
momentum=0.9,
weight_decay=0.0001)
elif netD_params['optim'] == 'Adam':
self.optimD = torch.optim.Adam(self.netD.parameters(),
lr=netD_params['lr'],
betas=(netD_params['beta1'], 0.999))
else:
raise Exception("No such optimizer: {}".format(
netD_params['optim']))
cudnn.benchmark = True
def __init__(self, final_caps_len, final_caps_num, in_channels, in_height,
in_width):
super(CapsNetDecoder, self).__init__()
self.in_channels = in_channels
self.in_height = in_height
self.in_width = in_width
self.reconstruct = nn.Sequential(
init_weights(nn.Linear(final_caps_len * final_caps_num, 512)),
nn.ReLU(inplace=True), init_weights(nn.Linear(512, 1024)),
nn.ReLU(inplace=True),
init_weights(nn.Linear(1024, in_channels * in_height * in_width)),
nn.Sigmoid())
def __init__(self, in_size, out_size, is_deconv, n_concat=2):
super(unetUp, self).__init__()
self.conv = unetConv2(in_size+(n_concat-2)*out_size, out_size, False)
if is_deconv:
self.up = nn.ConvTranspose2d(in_size, out_size, kernel_size=2, stride=2, padding=0)
else:
self.up = nn.Sequential(
nn.UpsamplingBilinear2d(scale_factor=2),
nn.Conv2d(in_size, out_size, 1))
# initialise the blocks
for m in self.children():
if m.__class__.__name__.find('unetConv2') != -1: continue
init_weights(m, init_type='kaiming')
def __init__(self,
losses=dict(adv=1,
rec=100,
self=100,
triple=100,
tv=1e-5,
percep=0),
gradclip=0,
gantype='wgan-gp',
edge=False):
self.npoints = 68
self.gradclip = gradclip
self.l = losses
# - define models
self.DIS = Discriminator(ndim=0)
self.GEN = Generator(conv_dim=64,
c_dim=self.npoints if not edge else 3)
init_weights(self.DIS)
init_weights(self.GEN)
if torch.cuda.device_count() > 1:
self.GEN = torch.nn.DataParallel(self.GEN)
self.DIS = torch.nn.DataParallel(self.DIS)
self.DIS.to('cuda').train()
self.GEN.to('cuda').train()
if self.l['percep'] > 0:
# - VGG for perceptual loss
self.loss_network = LossNetwork(
torch.nn.DataParallel(vgg16(pretrained=True))
) if torch.cuda.device_count() > 1 else LossNetwork(
vgg16(pretrained=True))
self.loss_network.eval()
self.loss_network.to('cuda')
self.TripleLoss = (lambda x, y: torch.mean(torch.abs(x - y))
) if self.l['triple'] > 0 else None
self.SelfLoss = torch.nn.L1Loss().to(
'cuda') if self.l['self'] > 0 else None
self.RecLoss = torch.nn.L1Loss().to(
'cuda') if self.l['rec'] > 0 else None
self.PercepLoss = torch.nn.MSELoss().to(
'cuda') if self.l['percep'] > 0 else None
self.TVLoss = TVLoss if self.l['tv'] > 0 else None
self.gantype = gantype
self.loss = dict(G=dict.fromkeys(
['adv', 'self', 'triple', 'percep', 'rec', 'tv', 'all']),
D=dict.fromkeys(['real', 'fake', 'gp', 'all']))
def param_init_sgmod(params, prefix, units, zero_init=True):
'''
Initialization for synthetic gradient subnetwork
'''
global args
# conditioned on the whole image, on the activation produced by encoder input and the backpropagated gradients for latent samples.
inp_list = [14*28, 14*28, units, units, units]
inp_size = 0
for i in range(5):
inp_size += inp_list[i]
if not zero_init:
if args.sg_type == 'lin':
params[_concat(prefix, 'W')] = init_weights(inp_size, units, type_init='ortho')
params[_concat(prefix, 'b')] = np.zeros((units,)).astype('float32')
else:
if args.sg_type == 'lin' or args.sg_type == 'lin_deep':
params[_concat(prefix, 'W')] = np.zeros((inp_size, units)).astype('float32')
params[_concat(prefix, 'b')] = np.zeros((units,)).astype('float32')
if args.sg_type == 'deep' or args.sg_type == 'lin_deep':
params = param_init_fflayer(params, _concat(prefix, 'I'), inp_size, 1024, batchnorm=True)
params = param_init_fflayer(params, _concat(prefix, 'H'), 1024, 1024, batchnorm=True)
if args.bn_type == 0:
params = param_init_fflayer(params, _concat(prefix, 'o'), 1024, units, zero_init=True, batchnorm=True)
else:
params = param_init_fflayer(params, _concat(prefix, 'o'), 1024, units, zero_init=True, batchnorm=False)
return params
def param_init_fflayer(params,
prefix,
nin,
nout,
zero_init=False,
batchnorm=False,
skip_running_vars=False):
'''
Initializes weights for a feedforward layer
'''
global args
if zero_init:
params[_concat(prefix, 'W')] = np.zeros((nin, nout)).astype('float32')
else:
params[_concat(prefix, 'W')] = init_weights(nin,
nout,
type_init='ortho')
params[_concat(prefix, 'b')] = np.zeros((nout, )).astype('float32')
if batchnorm:
if args.bn_type == 0:
dim = nin
else:
dim = nout
params[_concat(prefix, 'g')] = np.ones((dim, ), dtype=np.float32)
params[_concat(prefix, 'be')] = np.zeros((dim, )).astype('float32')
# it is not necessary for deep synthetic subnetworks to track running averages as they are not used in test time
if not skip_running_vars:
params[_concat(prefix, 'rm')] = np.zeros(
(1, dim)).astype('float32')
params[_concat(prefix, 'rv')] = np.ones((1, dim), dtype=np.float32)
return params
def cv_fit_model(model, X_train, y_train, nb_epoch=2, weights=False):
"""
Fit RNN model with ...
:param model:
:param X_train:
:param y_train:
:param nb_epoch:
:param weights:
:return:
"""
if weights:
sample_weights = init_weights(y_train)
else:
sample_weights = np.ones_like(y_train)
for i in range(nb_epoch):
print "Epoch " + str(i) + '/' + str(nb_epoch)
model.fit(X_train,
y_train,
batch_size=1024,
nb_epoch=1,
shuffle=False,
sample_weight=sample_weights
)
clear_output(wait=True)
return model
def init_models(self):
init_weights(self.gen_mtp)
init_weights(self.gen_ptm)
init_weights(self.dis_m)
init_weights(self.dis_p)
self.gen_mtp = self.gen_mtp.to(self.device) # G
self.gen_ptm = self.gen_ptm.to(self.device) # F
self.dis_m = self.dis_m.to(self.device)
self.dis_p = self.dis_p.to(self.device)
def create_model(config, dataset, create_W_emb=False):
model_class = None
model_params = {}
if config.model == 'counts':
model_class = CountsModel
model_params.update(
dict(
input_size=dataset.nb_features,
output_size=dataset.nb_classes,
dropout=config.dropout,
))
elif config.model == 'han' or config.model == 'rnn':
if create_W_emb:
word_embeddings = load_pickle(EMBEDDINGS_FILENAME)
print(f'Embeddings: {len(word_embeddings)}')
W_emb = create_embeddings(word_embeddings, dataset.vocab)
else:
W_emb = None
model_class = RNNModel
model_params.update(
dict(vocab_size=dataset.vocab_size,
trainable_embeddings=config.trainable_embeddings,
hidden_size=config.hidden_size,
dropout=config.dropout,
output_size=dataset.nb_classes,
W_emb=W_emb,
embedding_size=300,
padding_idx=0))
if config.model == 'han':
model_class = HANModel
model_params.update(dict(attention_size=config.attention_size, ))
else:
raise ValueError(f'Model {config.model} is unknown')
model = model_class(**model_params)
init_weights(model)
model = to_device(model)
print(f'Model: {model.__class__.__name__}')
return model
def param_init_fflayer(params, prefix, nin, nout):
'''
Initializes weights for a feedforward layer
'''
params[_concat(prefix, 'W')] = init_weights(nin, nout, type_init='ortho')
params[_concat(prefix, 'b')] = np.zeros((nout, )).astype('float32')
return params
def init(self):
# initialze the weigths for the LSTM RNN
# group Wi, Wf, Wo and Wc into a W matrix with 4 times the original number of columns
# group bi, bf, bo and bc into a b matrix with 4 times the original number of columns
# group Whi, Whf, Who and Whc into a Wh matrix with 4 times the original number of columns
# this allows a better usage of the vector units in the CPU/GPU
self.W, self.Wh, self.b = [], [], []
for i in range(len(self.rnn_layers)):
n_inputs = self.vocab_size if i == 0 else self.rnn_layers[i-1]
self.W.append(init_weights((n_inputs, 4 * self.rnn_layers[i]), rng=self.numpy_rng, name='W_{}'.format(i)))
self.Wh.append(init_ortho(self.rnn_layers[i], num_matrices=4, rng=self.numpy_rng, name='Wh_{}'.format(i)))
self.b.append(theano.shared(value=np.zeros((4 * self.rnn_layers[i],), dtype=theano.config.floatX), borrow=True, name='b_{}'.format(i)))
self.Why = init_weights((self.rnn_layers[i], self.vocab_size), rng=self.numpy_rng, name='Why') # hidden to output weights
self.by = theano.shared(value=np.zeros((self.vocab_size,), dtype=theano.config.floatX), borrow=True, name='by') # hidden to output bias
params = self.W + self.Wh + self.b + [self.Why, self.by]
return params
def build_model(self):
# Weights for fully connected layers
self.w_alice = init_weights("alice_w", [self.msg_len + self.secret_len + self.key_len, 2*self.msg_len])
self.w_bob = init_weights("bob_w", [self.msg_len + self.key_len, 2 * self.secret_len])
self.w_keygen = init_weights("keygen_w",[self.random_seed_len, 2*self.key_len])
self.w_eve1 = init_weights("eve_w1", [self.msg_len, 2 * self.msg_len])
self.w_eve2 = init_weights("eve_w2", [2 * self.msg_len, 2 * self.secret_len])
# Placeholder variables for Message and Key
self.msg = tf.placeholder("float", [None, self.msg_len])
self.secret = tf.placeholder("float", [None, self.secret_len])
self.seed = tf.placeholder("float", [None, self.random_seed_len])
# KeyGen's network
# self.keygen_input = self.seed
# self.keygen_hidden = tf.nn.tanh(tf.matmul(self.keygen_input,self.w_keygen))
# self.keygen_hidden = tf.expand_dims(self.keygen_hidden, 2)
# self.key = tf.sigmoid(tf.squeeze(conv_layer(self.keygen_hidden,"keygen")));
self.key = self.seed
# Alice's network
# FC layer -> Conv Layer (4 1-D convolutions)
self.alice_input = tf.concat(axis=1, values=[self.msg, self.secret, self.key])
self.alice_hidden = tf.nn.tanh(tf.matmul(self.alice_input, self.w_alice))
self.alice_hidden = tf.expand_dims(self.alice_hidden, 2)
self.alice_output = tf.squeeze(conv_layer(self.alice_hidden, "alice"))
#self.alice_output = encrypt(self.msg,self.key)
# Bob's network
# FC layer -> Conv Layer (4 1-D convolutions)
self.bob_input = tf.concat(axis=1, values=[self.alice_output, self.key])
self.bob_hidden = tf.nn.tanh(tf.matmul(self.bob_input, self.w_bob))
self.bob_hidden = tf.expand_dims(self.bob_hidden, 2)
self.bob_output = tf.squeeze(conv_layer(self.bob_hidden, "bob"))
#self.bob_output = decrypt(self.alice_output,self.key)
# Eve's network
# FC layer -> FC layer -> Conv Layer (4 1-D convolutions)
self.eve_input = self.alice_output
self.eve_hidden1 = tf.nn.tanh(tf.matmul(self.eve_input, self.w_eve1)) #Sigmoid Earlier
self.eve_hidden2 = tf.nn.tanh(tf.matmul(self.eve_hidden1, self.w_eve2)) #Sigmoid Earlier
self.eve_hidden2 = tf.expand_dims(self.eve_hidden2, 2)
self.eve_output = tf.squeeze(conv_layer(self.eve_hidden2, "eve"))
def create_model(test=False, log_dir=None, pretrain=False):
model = torchvision.models.segmentation.fcn_resnet50(False)
model.classifier[4] = nn.Conv2d(512, 1, kernel_size=(1, 1), stride=(1, 1))
if test:
if not log_dir.endswith('pth'):
log_dir = log_dir + '/final_model_checkpoint.pth'
if os.path.exists(log_dir):
model = init_weights(model, log_dir)
return model
def __init__(self, in_channels, out_channels, vec_len):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.vector_length = vec_len
conv = nn.Conv2d(in_channels=in_channels, out_channels=out_channels * vec_len, kernel_size=9, stride=2,
bias=True)
self.conv = init_weights(conv)
def init(self): # initialze the weigths for the Vanilla RNN self.Wxh, self.Whh, self.bh = [], [], [] # input-to-hidden weights, hidden-to-hidden weights and biases # TODO: initialize the parameters of the RNN. Use the functions defined in the utils package self.Why = utils.init_weights((self.rnn_layers[i], self.vocab_size), rng=self.numpy_rng, name='Why') # hidden to output weights self.by = theano.shared(value=np.zeros((self.vocab_size,), dtype=theano.config.floatX), borrow=True, name='by') # hidden to output bias params = self.Wxh + self.Whh + self.bh + [self.Why, self.by] return params
def init(self):
# initialze the weigths for the Vanilla RNN
self.Wxh, self.Whh, self.bh, self.by = [], [], [], []
for i in range(len(self.rnn_layers)):
n_inputs = self.vocab_size if i == 0 else self.rnn_layers[i-1]
self.Wxh.append(utils.init_weights((n_inputs, self.rnn_layers[i]),
rng=self.numpy_rng, name='Wxh_{}'.format(i))) # input to hidden weigths
self.Whh.append(utils.init_ortho(self.rnn_layers[i],
rng=self.numpy_rng, name='Whh_{}'.format(i))) # hidden to hidden weights
self.bh.append(theano.shared(value=np.zeros((self.rnn_layers[i],),
dtype=theano.config.floatX), borrow=True, name='bh_{}'.format(i))) # hidden to hidden bias
self.Why = utils.init_weights((self.rnn_layers[i], self.vocab_size),
rng=self.numpy_rng, name='Why') # hidden to output weights
self.by = theano.shared(value=np.zeros((self.vocab_size,),
dtype=theano.config.floatX), borrow=True, name='by') # hidden to output bias
params = self.Wxh + self.Whh + self.bh + [self.Why, self.by]
return params
def build_model(self, netpath: str = None):
if self.outchannel is None:
self.outchannel = self.img_.shape[1]
if self.args.net == "load":
_args = u.read_args(os.path.join('results', *netpath.split('/')[:-1], "args.txt"))
assert net_args_are_same(self.args, _args)
self.net = get_net(_args, self.outchannel).type(self.dtype)
self.net.load_state_dict(torch.load(os.path.join('results', netpath)))
else:
self.net = get_net(self.args, self.outchannel).type(self.dtype)
u.init_weights(self.net, self.args.inittype, self.args.initgain)
# self.net = self.net.type(self.dtype)
#
# if self.args.net != 'load':
# u.init_weights(self.net, self.args.inittype, self.args.initgain)
self.parameters = u.get_params('net', self.net, self.input_)
self.num_params = sum(np.prod(list(p.size())) for p in self.net.parameters())
def param_init_sgmod(params, prefix, units, zero_init=True):
'''
Initializes a linear regression based model for estimating gradients, conditioned on the class labels
'''
if not zero_init:
params[_concat(prefix, 'W')] = init_weights(units,
units,
type_init='ortho')
params[_concat(prefix, 'C')] = init_weights(10,
units,
type_init='ortho')
else:
params[_concat(prefix, 'W')] = np.zeros(
(units, units)).astype('float32')
params[_concat(prefix, 'C')] = np.zeros((10, units)).astype('float32')
params[_concat(prefix, 'b')] = np.zeros((units, )).astype('float32')
return params
def __init__(self,
feature_dim=784,
latent_dim=10,
input_size=(1, 28, 28),
verbose=False):
super(Classifier_CNN, self).__init__()
self.feature_dim = feature_dim
self.latent_dim = latent_dim
self.input_size = input_size
self.cshape = (128, 6, 6)
self.iels = int(np.prod(self.cshape))
self.lshape = (self.iels, )
self.vervose = verbose
self.model = nn.Sequential(
nn.Conv2d(1, 64, kernel_size=4, stride=1, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(True),
nn.MaxPool2d(2),
nn.Conv2d(64, 128, kernel_size=4, stride=1, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(True),
nn.MaxPool2d(2),
)
self.model = nn.Sequential(
self.model,
Reshape(self.lshape),
nn.Linear(self.iels, 1024),
nn.BatchNorm1d(1024),
nn.ReLU(True),
nn.Linear(1024, self.latent_dim),
# nn.Softmax(),
)
init_weights(self)
if self.vervose:
print(self.model)
def __init__(self, image_channels=1, image_size=28):
super(Discriminator, self).__init__()
self.image_channels = image_channels
self.image_size = image_size
self.conv = nn.Sequential(
# input a real image, going to a convolution
nn.Conv2d(self.image_channels, 64, 4, 2, 1), # (-1, 64, 14 ,14)
nn.LeakyReLU(0.2),
nn.Conv2d(64, 128, 4, 2, 1), # (-1, 128, 7, 7)
nn.BatchNorm2d(128),
nn.LeakyReLU(0.2))
# reshape -> (-1, 128*7*7)
self.fc = nn.Sequential(
nn.Linear(128 * (self.image_size // 4) * (self.image_size // 4),
1024), # (-1, 1024)
nn.BatchNorm1d(1024),
nn.LeakyReLU(0.2),
nn.Linear(1024, 1), # (-1, 1)
nn.Sigmoid())
init_weights(self)
self._print_info()
def __init__(self, parsed_args, parsed_groups):
super().__init__(**parsed_groups['trainer arguments'])
self.gen = UNetGenerator(**parsed_groups['generator arguments'])
self.disc = NLayerDiscriminator(
**parsed_groups['discriminator arguments'])
init_weights(self.gen)
init_weights(self.disc)
self.real_label = torch.tensor(1.0)
self.fake_label = torch.tensor(0.0)
self.crit = torch.nn.MSELoss() # LSGAN
self.image_pool = ImagePool(parsed_args.pool_size,
parsed_args.replay_prob)
self.sel_ind = 0
self.un_normalize = lambda x: 255. * (1 + x.clamp(min=-1, max=1)) / 2.
self.parsed_args = parsed_args
self.n_vis = parsed_args.n_vis
self.vis = Visualizer()