def main(args):
if not os.path.exists(args.conf):
print('Config not found' + args.conf)
config = read_config(args.conf)
print(colored(str(config), 'cyan'))
eval_flag = config['eval']
if not eval_flag: #train mode : fresh or reload
current_log_dir = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
current_log_dir = os.path.join('../Experiments/', current_log_dir)
else: #eval mode : save result plys
if args.load_checkpoint_dir:
current_log_dir = '../Eval'
else:
print(
colored(
'*****please provide checkpoint file path to reload!*****',
'red'))
return
print(colored('logs will be saved in:{}'.format(current_log_dir),
'yellow'))
if args.load_checkpoint_dir:
load_checkpoint_dir = os.path.join('../Experiments/',
args.load_checkpoint_dir,
'chkpt') #load last checkpoint
print(
colored('load_checkpoint_dir: {}'.format(load_checkpoint_dir),
'red'))
save_checkpoint_dir = os.path.join(current_log_dir, 'chkpt')
print(
colored('save_checkpoint_dir: {}\n'.format(save_checkpoint_dir),
'yellow'))
if not os.path.exists(save_checkpoint_dir):
os.makedirs(save_checkpoint_dir)
print('Initializing parameters')
template_file_path = config['template_fname']
template_mesh = Mesh(filename=template_file_path)
print(template_file_path)
visualize = config['visualize']
lr = config['learning_rate']
lr_decay = config['learning_rate_decay']
weight_decay = config['weight_decay']
total_epochs = config['epoch']
workers_thread = config['workers_thread']
opt = config['optimizer']
batch_size = config['batch_size']
val_losses, accs, durations = [], [], []
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
print(colored('\n...cuda is available...\n', 'green'))
else:
print(colored('\n...cuda is NOT available...\n', 'red'))
ds_factors = config['downsampling_factors']
print('Generating transforms')
M, A, D, U = mesh_operations.generate_transform_matrices(
template_mesh, ds_factors)
D_t = [scipy_to_torch_sparse(d).to(device) for d in D]
U_t = [scipy_to_torch_sparse(u).to(device) for u in U]
A_t = [scipy_to_torch_sparse(a).to(device) for a in A]
num_nodes = [len(M[i].v) for i in range(len(M))]
print(colored('number of nodes in encoder : {}'.format(num_nodes), 'blue'))
if args.data_dir:
data_dir = args.data_dir
else:
data_dir = config['data_dir']
print('*** data loaded from {} ***'.format(data_dir))
dataset = ComaDataset(data_dir,
dtype='train',
split=args.split,
split_term=args.split_term)
dataset_test = ComaDataset(data_dir,
dtype='test',
split=args.split,
split_term=args.split_term)
train_loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers_thread)
test_loader = DataLoader(dataset_test,
batch_size=1,
shuffle=False,
num_workers=workers_thread)
print("x :\n{} for dataset[0] element".format(dataset[0]))
print(colored(train_loader, 'red'))
print('Loading Model : \n')
start_epoch = 1
coma = Coma(dataset, config, D_t, U_t, A_t, num_nodes)
tbSummWriter = SummaryWriter(current_log_dir)
print_model_summary = False
if print_model_summary:
print(coma)
mrkdwn = str('<pre><code>' + str(coma) + '</code></pre>')
tbSummWriter.add_text('tag2', mrkdwn, global_step=None, walltime=None)
#write network architecture into text file
logfile = os.path.join(current_log_dir, 'coma.txt')
my_data_file = open(logfile, 'w')
my_data_file.write(str(coma))
my_data_file.close()
if opt == 'adam':
optimizer = torch.optim.Adam(coma.parameters(),
lr=lr,
weight_decay=weight_decay)
elif opt == 'sgd':
optimizer = torch.optim.SGD(coma.parameters(),
lr=lr,
weight_decay=weight_decay,
momentum=0.9)
else:
raise Exception('No optimizer provided')
if args.load_checkpoint_dir:
#to load the newest saved checkpoint
to_back = os.getcwd()
os.chdir(load_checkpoint_dir)
chkpt_list = sorted(os.listdir(os.getcwd()), key=os.path.getctime)
os.chdir(to_back)
checkpoint_file = chkpt_list[-1]
logfile = os.path.join(current_log_dir, 'loadedfrom.txt')
my_data_file = open(logfile, 'w')
my_data_file.write(str(load_checkpoint_dir))
my_data_file.close()
print(
colored(
'\n\nloading Newest checkpoint : {}\n'.format(checkpoint_file),
'red'))
if checkpoint_file:
checkpoint = torch.load(
os.path.join(load_checkpoint_dir, checkpoint_file))
start_epoch = checkpoint['epoch_num']
coma.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
#To find if this is fixed in pytorch
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.to(device)
coma.to(device)
for i, dt in enumerate(train_loader):
dt = dt.to(device)
graphstr = pms.summary(coma,
dt,
batch_size=-1,
show_input=True,
show_hierarchical=False)
if print_model_summary:
print(graphstr)
print(colored('dt in enumerate(train_loader):{} '.format(dt), 'green'))
#write network architecture into text file
logfile = os.path.join(current_log_dir, 'pms.txt')
my_data_file = open(logfile, 'w')
my_data_file.write(graphstr)
my_data_file.close()
mrkdwn = str('<pre><code>' + graphstr + '</code></pre>')
tbSummWriter.add_text('tag', mrkdwn, global_step=None, walltime=None)
break #for one sample only
if eval_flag and args.load_checkpoint_dir:
evaluatedFrom = 'predictedPlys_' + checkpoint_file
output_dir = os.path.join('../Experiments/', args.load_checkpoint_dir,
evaluatedFrom) #load last checkpoint
val_loss = evaluate(coma,
test_loader,
dataset_test,
template_mesh,
device,
visualize=True,
output_dir=output_dir)
print('val loss', val_loss)
return
best_val_loss = float('inf')
val_loss_history = []
for epoch in range(start_epoch, total_epochs + 1):
print("Training for epoch ", epoch)
print('dataset.len : {}'.format(len(dataset)))
train_loss = train(coma, train_loader, len(dataset), optimizer, device)
val_loss = evaluate(coma,
test_loader,
dataset_test,
template_mesh,
device,
visualize=False,
output_dir='') #train without visualization
tbSummWriter.add_scalar('Loss/train', train_loss, epoch)
tbSummWriter.add_scalar('Val Loss/train', val_loss, epoch)
tbSummWriter.add_scalar('learning_rate', lr, epoch)
print('epoch ', epoch, ' Train loss ', train_loss, ' Val loss ',
val_loss)
if val_loss < best_val_loss:
save_model(coma, optimizer, epoch, train_loss, val_loss,
save_checkpoint_dir)
best_val_loss = val_loss
val_loss_history.append(val_loss)
val_losses.append(best_val_loss)
if opt == 'sgd':
adjust_learning_rate(optimizer, lr_decay)
if torch.cuda.is_available():
torch.cuda.synchronize()
tbSummWriter.flush()
tbSummWriter.close()
y : tensor
Output of shape (N, seq_len, num_channels[-1]).
"""
y = self.network(x)
if self.attention is not None:
y = self.attention(y)
y = y.reshape(y.shape[0], -1)
return self.fc(y)
if __name__ == '__main__':
# model = Chomp1d(3)
# x = torch.randn(128, 12, 24)
# print(model(x).shape)
# model = TemporalBlock(n_inputs=14, n_outputs=1, kernel_size=2, stride=1, dilation=1, padding=1)
model = TemporalConvNet(10, 1, (16,128, 1), 12)
# model = nn.Sequential(AttentionBlock(10,10,10))
# # model = Transpose_Layer(1,2)
x = torch.randn(19, 12, 10)
# # print(x)
# print(model(x).shape)
# # x = x.transpose(1,2)
# # model.training = True
# # print(model(x)[0].shape,model(x)[1].shape, model.training)
print(summary(model, x , show_hierarchical=False, show_input=True, show_parent_layers=True))
print(summary(model, x , show_hierarchical=False, show_input=False, show_parent_layers=True))
# # m = Conv2d_Block(3, 16, kernel_size=(2,1), padding=(2,1))
# # print(m(x).shape)
f.write('--------------------------------\n')
if not os.path.isdir(args.destination_dir):
print('destination directory not exists. Exiting...')
raise error('destination directory not exists.')
path = os.path.join(args.destination_dir, 'cfg', 'net_arch.pickle')
with open(path, 'rb') as handle:
network_architecture = pickle.load(handle)
path = os.path.join(args.destination_dir, 'cfg', 'trn_para.pickle')
with open(path, 'rb') as handle:
tps = pickle.load(handle)
model = get_model(network_architecture)
print(summary(model, torch.zeros(1, 3, 32, 32), show_input=False))
# initialize loaders
torch.manual_seed(tps['seed'])
#if tps['data_name'] == 'cifar100':
# ds = DS_cifar100(tps['data_name'], tps['data_path'], tps['batch_size'], tps['valid_size'], 'cfg/data', tps['data_indices_path'])
#elif tps['data_name'] == 'cifar10':
# ds = DS_cifar10(tps['data_name'], tps['data_path'], tps['batch_size'], tps['valid_size'], 'cfg/data', tps['data_indices_path'])
ds = get_dataset(tps['data_name'], tps['data_path'], tps['batch_size'],
tps['valid_size'], 'cfg/data', tps['data_indices_path'])
# Begin train
train(ds, network_architecture, tps, args.destination_dir + '/outputs',
args.device_type)
# Begin calibrate
calibrate(ds, network_architecture, tps, args.destination_dir + '/outputs',
args.device_type)
optimizer = optim.Adam(
[param for param in model.parameters() if param.requires_grad == True],
lr=config.lr,
eps=config.eps,
weight_decay=config.weight_decay)
writer.add_graph(
model,
torch.randint(low=0,
high=100,
size=(config.batch_size, 32),
dtype=torch.long).cuda())
print(
summary(
model,
torch.randint(low=0,
high=100,
size=(config.batch_size, 32),
dtype=torch.long).cuda()))
def generate_batch(batch):
# TextDataSet yield one line contain label and input
batch_label, batch_input = [], []
for data in batch:
num_input = []
if len(data.split('\t')) != 2:
print(data)
label, sent = data.split('\t')
#print(sent1, sent2, label)
batch_label.append(label2id[label])
sent = torch.squeeze(bert_tokenizer.encode(
def _document_models(self) -> None:
"""Add model summaries to the traceability document.
"""
with self.doc.create(Section("Models")):
for model in humansorted(self.system.network.models,
key=lambda m: m.model_name):
if not isinstance(model, (tf.keras.Model, torch.nn.Module)):
continue
self.doc.append(NoEscape(r'\FloatBarrier'))
with self.doc.create(
Subsection(f"{model.model_name.capitalize()}")):
if isinstance(model, tf.keras.Model):
# Text Summary
summary = []
model.summary(line_length=92,
print_fn=lambda x: summary.append(x))
summary = "\n".join(summary)
self.doc.append(Verbatim(summary))
with self.doc.create(Center()):
self.doc.append(
HrefFEID(FEID(id(model)), model.model_name))
# Visual Summary
# noinspection PyBroadException
try:
file_path = os.path.join(
self.resource_dir,
"{}_{}.pdf".format(self.report_name,
model.model_name))
dot = tf.keras.utils.model_to_dot(
model, show_shapes=True, expand_nested=True)
# LaTeX \maxdim is around 575cm (226 inches), so the image must have max dimension less than
# 226 inches. However, the 'size' parameter doesn't account for the whole node height, so
# set the limit lower (100 inches) to leave some wiggle room.
dot.set('size', '100')
dot.write(file_path, format='pdf')
except Exception:
file_path = None
print(
f"FastEstimator-Warn: Model {model.model_name} could not be visualized by Traceability"
)
elif isinstance(model, torch.nn.Module):
if hasattr(model, 'fe_input_spec'):
# Text Summary
# noinspection PyUnresolvedReferences
inputs = model.fe_input_spec.get_dummy_input()
self.doc.append(
Verbatim(
pms.summary(
model.module if
self.system.num_devices > 1 else model,
inputs,
print_summary=False)))
with self.doc.create(Center()):
self.doc.append(
HrefFEID(FEID(id(model)),
model.model_name))
# Visual Summary
# Import has to be done while matplotlib is using the Agg backend
old_backend = matplotlib.get_backend() or 'Agg'
matplotlib.use('Agg')
# noinspection PyBroadException
try:
# Fake the IPython import when user isn't running from Jupyter
sys.modules.setdefault('IPython', MagicMock())
sys.modules.setdefault('IPython.display',
MagicMock())
import hiddenlayer as hl
with Suppressor():
graph = hl.build_graph(
model.module if
self.system.num_devices > 1 else model,
inputs)
graph = graph.build_dot()
graph.attr(
rankdir='TB'
) # Switch it to Top-to-Bottom instead of Left-to-Right
# LaTeX \maxdim is around 575cm (226 inches), so the image must have max dimension less
# than 226 inches. However, the 'size' parameter doesn't account for the whole node
# height, so set the limit lower (100 inches) to leave some wiggle room.
graph.attr(size="100,100")
graph.attr(margin='0')
file_path = graph.render(
filename="{}_{}".format(
self.report_name, model.model_name),
directory=self.resource_dir,
format='pdf',
cleanup=True)
except Exception:
file_path = None
print(
"FastEstimator-Warn: Model {} could not be visualized by Traceability"
.format(model.model_name))
finally:
matplotlib.use(old_backend)
else:
file_path = None
self.doc.append(
"This model was not used by the Network during training."
)
if file_path:
with self.doc.create(Figure(position='ht!')) as fig:
fig.append(
Label(
Marker(name=str(FEID(id(model))),
prefix="model")))
fig.add_image(
os.path.relpath(file_path,
start=self.save_dir),
width=NoEscape(
r'1.0\textwidth,height=0.95\textheight,keepaspectratio'
))
fig.add_caption(
NoEscape(
HrefFEID(FEID(id(model)),
model.model_name).dumps()))
backbone_out_features = dict(regnetx_004=384,
regnety_004=440)[model_achitecture]
elif model_achitecture == "densenet121":
backbone = timm.create_model("densenet121",
pretrained=True,
in_chans=num_in_channels)
backbone_out_features = 1024
else:
raise RuntimeError("Invalid model_achitecture:", model_achitecture)
return backbone, backbone_out_features
if __name__ == "__main__":
import pytorch_model_summary
nb_raster_channels = 13
backbone, size = build_backbone("hrnet_w48", nb_raster_channels)
print(
pytorch_model_summary.summary(
backbone,
torch.zeros((1, nb_raster_channels, 288, 288)),
show_input=False,
show_hierarchical=False,
print_summary=True,
))
out = self.a5(out)
out = self.b5(out)
out = self.avgpool(out)
out = out.view(out.size(0), -1)
out = self.linear(out)
return out
def test():
net = GoogLeNet()
x = torch.randn(1, 3, 32, 32)
y = net(x)
print(y.size())
# test()
from pytorch_model_summary import summary
model_name = 'GoogLeNet'
print(model_name)
net = GoogLeNet()
batch_size = 1
print(
summary(net,
torch.randn(batch_size, 3, 32, 32),
batch_size=batch_size,
show_input=True,
show_hierarchical=True,
max_depth=True,
show_parent_layers=True))
def train_on_fold(model, fold_id, feature_name, objective_type, args):
torch.manual_seed(0)
device = get_torch_device(args)
results = {}
train_loader, test_loader, train_bias_category_labels, test_bias_category_labels = get_loaders_for_fold(
fold_id, feature_name, args.batch_size, args)
logging.info("Model Summary :\n" +
summary(model,
torch.zeros((10, args.max_sequence_length,
model.input_channels)).to(device),
show_input=False))
logging.info(f"train_n: {len(train_loader.dataset)}")
logging.info(f"test_n: {len(test_loader.dataset)}")
optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
criterion = nn.CrossEntropyLoss(reduction='sum')
for epoch in range(1, args.epochs + 1):
# train on training set
train(model, optimizer, criterion, objective_type, train_loader)
# test on training set
train_n, train_average_loss, train_acc, train_acc_by_bais_category, train_acc_lng, train_acc_by_bais_category_lng = \
test(model, criterion, objective_type, train_loader, train_bias_category_labels)
# test on test set
test_n, test_average_loss, test_acc, test_acc_by_bais_category, test_acc_lng, test_acc_by_bais_category_lng = \
test(model, criterion, objective_type, test_loader, test_bias_category_labels)
# persist
if args.persistence_interval > 0 and epoch % args.persistence_interval == 0:
models.save_model(model, optimizer, fold_id, feature_name, epoch,
args)
if epoch % 10 == 0:
logging.info(
f"Epoch: {epoch}. Train Loss: {train_average_loss:0.4}. Test Loss: {test_average_loss:0.4}. Train Acc: {train_acc:0.4}. Test Acc:{test_acc:0.4}"
)
else:
logging.debug(
f"Epoch: {epoch}. Train Loss: {train_average_loss:0.4}. Test Loss: {test_average_loss:0.4}. Train Acc: {train_acc:0.4}. Test Acc:{test_acc:0.4}"
)
results[epoch] = {
'epoch': epoch,
'train_n': train_n,
'train_loss': train_average_loss,
'train_acc': train_acc,
'train_acc_lng': train_acc_lng,
'test_n': test_n,
'test_loss': test_average_loss,
'test_acc': test_acc,
'test_acc_lng': test_acc_lng
}
for c in train_acc_by_bais_category:
results[epoch][f"train_acc_{c}"] = train_acc_by_bais_category[c]
results[epoch][f"train_n_{c}"] = int(
np.sum(train_bias_category_labels[c]))
for c in train_acc_by_bais_category_lng:
results[epoch][
f"train_acc_lng_{c}"] = train_acc_by_bais_category_lng[c]
for c in test_acc_by_bais_category:
results[epoch][f"test_acc_{c}"] = test_acc_by_bais_category[c]
results[epoch][f"test_n_{c}"] = int(
np.sum(test_bias_category_labels[c]))
for c in test_acc_by_bais_category_lng:
results[epoch][
f"test_acc_lng_{c}"] = test_acc_by_bais_category_lng[c]
return results
networks = NetworkApi(noRun=True)
net = ResCNN(6,
7,
1,
3,
128,
32,
3,
7,
3,
1,
mode="sq",
outputExtra="bothhead")
# net = ResCNN(6, 7, 1, 3, 64, 16, 1, 7, 3, 8, mode="sq", outputExtra="bothhead")
print(pms.summary(net, torch.zeros((1, 1, 6, 7))))
networkData = networks.downloadNetwork(networkID)
uuid, modelDict, ignoreConfig = unpackTorchNetwork(networkData)
# net.load_state_dict(modelDict)
game = createGame([4, 4, 4, 4, 4, 4, 5, 3, 6, 7])
# game = createGame([])
# game = createGame([4,4,3,4,2])
print(str(game))
forwardGameState(net, game)
def summary(self, model):
if self.torchsummary:
example = model.example_input_array
summary(model, example, print_summary=True, max_depth=self.depth)
out_channels=32,
kernel_size=3,
stride=(2, 1),
padding=(0, 1))
self.FC = nn.Linear(in_features=32 * 31,
out_features=256) # last conv to rnn in
self.RNN = BiRNN(input_size=256, hidden_size=256, output_size=256)
self.classifier = Classifier(256, 128, num_classes=29) # 29 characters
def forward(self, x): # shape (batch_size, frequencies=64, time)
x = x.unsqueeze(
1) # shape (batch_size, channels=1, frequencies=64, time)
x = self.conv0(
x) # shape (batch_size, channels=32, frequencies=64, time)
x = self.encoder(
x) # shape (batch_size, channels=32, frequencies=64, time)
x = self.shrink(
x) # shape (batch_size, channels=32, frequencies=31, time)
# unrolling to shape (batch_size, time, frequencies=31*channels=32)
x = transpose(
x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3]), 1, 2)
x = self.FC(x) # shape (batch_size, time, 256)
x = transpose(x, 1, 0) # shape (time, batch_size, 256)
x = self.RNN(x) # shape (time, batch_size, 256)
x = transpose(x, 1, 0) # shape (batch_size, time, 256)
return self.classifier(x) # shape (batch_size, time, 29)
if __name__ == '__main__':
print(summary(ParrotModel(), torch.zeros((1, 64, 100))))
def init_weight(self):
state_dict = modelzoo.load_url(resnet18_url)
self_state_dict = self.state_dict()
for k, v in state_dict.items():
if 'fc' in k: continue
self_state_dict.update({k: v})
self.load_state_dict(self_state_dict)
def get_params(self):
wd_params, nowd_params = [], []
for name, module in self.named_modules():
if isinstance(module, (nn.Linear, nn.Conv2d)):
wd_params.append(module.weight)
if not module.bias is None:
nowd_params.append(module.bias)
elif isinstance(module, nn.modules.batchnorm._BatchNorm):
nowd_params += list(module.parameters())
return wd_params, nowd_params
if __name__ == "__main__":
net = Resnet18()
x = torch.randn(16, 3, 224, 224)
out = net(x)
print(out[0].size())
print(out[1].size())
print(out[2].size())
print(summary(net, x, show_input=True))
make_dot(out)
net.get_params()
# do text parsing, get vocab size and class count
build_vocab(args.train, args.output_vocab_label, args.output_vocab_word)
label2id, id2label = load_vocab(args.output_vocab_label)
word2id, id2word = load_vocab(args.output_vocab_word)
vocab_size = len(word2id)
num_class = len(label2id)
# set model
model = AttentionBiLSTM(vocab_size=vocab_size, num_class=num_class, emb_dim=args.embedding_dim, emb_droprate=args.embedding_droprate, sequence_len=args.sequence_len, att_droprate=args.att_droprate, rnn_cell_hidden=args.rnn_cell_hidden, num_layers=args.num_layers, att_method=args.att_method)
model.build()
model.to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=1e-6)
writer.add_graph(model, torch.randint(low=0, high=1000, size=(args.batch_size, args.sequence_len), dtype=torch.long).to(device))
print(summary(model, torch.randint(low=0, high=1000, size=(args.batch_size, args.sequence_len), dtype=torch.long).to(device)))
# padding sequence with <PAD>
def padding(data, fix_length, pad, add_first="", add_last=""):
if add_first:
data.insert(0, add_first)
if add_last:
data.append(add_last)
pad_data = []
data_len = len(data)
for idx in range(fix_length):
if idx < data_len:
pad_data.append(data[idx])
else:
pad_data.append(pad)
return pad_data
def print_pmx(model, *args):
pms.summary(model, *args, print_summary=True)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = AdamW(model.parameters(), lr=config.lr, eps=config.eps)
writer.add_graph(model, [
torch.randint(low=0,
high=1000,
size=(config.batch_size, config.sequence_len),
dtype=torch.long).to(device),
torch.ones(size=(config.batch_size, config.sequence_len),
dtype=torch.long).to(device)
])
print(
summary(
model,
torch.randint(low=0,
high=1000,
size=(config.batch_size, config.sequence_len),
dtype=torch.long).to(device),
torch.ones(size=(config.batch_size, config.sequence_len),
dtype=torch.long).to(device)))
# bert tokenizer
tokenizer = BertTokenizer(vocab_file=config.vocab_file, do_lower_case=True)
def generate_batch(batch):
# TextDataSet yield one line contain label and input
batch_label, input_ids, attention_masks = [], [], []
for data in batch:
num_input = []
label, sentence = data.split('\t')
batch_label.append(label2id[label])
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
self.conv2_drop = nn.Dropout2d()
self.fc1 = nn.Linear(320, 50)
self.fc2 = nn.Linear(50, 10)
def forward(self, x):
x = F.relu(F.max_pool2d(self.conv1(x), 2))
x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
x = x.view(-1, 320)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return F.log_softmax(x, dim=1)
# show input shape
print(summary(Net(), torch.zeros((1, 1, 28, 28)), show_input=True))
# show output shape
print(summary(Net(), torch.zeros((1, 1, 28, 28)), show_input=False))
# show output shape and hierarchical view of net
print(
summary(Net(),
torch.zeros((1, 1, 28, 28)),
show_input=False,
show_hierarchical=True))
from torchvision import models, transforms
model = models.resnet18(pretrained=False)
model.fc = nn.Linear(model.fc.in_features, 2)
print(model.fc.weight.size())
print(model.fc.weight[0].size())
print(model.fc.weight[1].size())
# print(model.layer4)
print(
summary(model,
torch.zeros((1, 3, 224, 224)),
show_input=True,
show_hierarchical=True))
# Conv = [7, 7, 512]
# Weights = [512]
# Conv * Weights => CAM
image_tensor = torch.zeros((1, 3, 224, 224))
extractor = nn.Sequential(*[
model.conv1,
model.bn1,
model.relu,
model.maxpool,
model.layer1,
model.layer2,
x = x.view(-1, 64)
x = self.fc(x)
return x
def reset_weights_to_init(self):
self.load_state_dict(self.initial_state_dict, strict=False)
def reset_weights_to_random(self):
new_state_dict = {}
for k, v in self.initial_state_dict.items():
new_state_dict[k] = shuffle_tensor(v.clone())
self.load_state_dict(new_state_dict, strict=False)
model = Model()
print(summary(model, torch.zeros(1, 3, 32, 32)))
criterion = nn.CrossEntropyLoss()
val_criterion = nn.CrossEntropyLoss(reduction="sum")
def train(epochs, callback=None):
optimizer = torch.optim.Adam(model.parameters(), lr=1.2e-3)
running_average_alpha = 0.9
for epoch in range(epochs):
running_loss = 0
t = time.time()
from __future__ import print_function
from __future__ import division
import torch
from model import UWnet
from ptflops import get_model_complexity_info
from flopth import flopth
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = UWnet(num_layers=3).to(device)
flops, params = get_model_complexity_info(model, (3, 256, 256),
as_strings=True,
print_per_layer_stat=True)
print('Densenet: {:<30} {:<8}'.format('Computational complexity: ', flops))
print('Densenet: {:<30} {:<8}'.format('Number of parameters: ', params))
pytorch_total_params = sum(p.numel() for p in model.parameters())
trainable_pytorch_total_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print('Total - ', pytorch_total_params)
print('Trainable - ', trainable_pytorch_total_params)
from pytorch_model_summary import summary
print(
summary(UWnet(num_layers=3),
torch.zeros((64, 3, 3, 3)),
show_input=False,
show_hierarchical=False))
vacc_history.append(vacc)
print(f'Epoch : {i+1} => train loss={tloss:.4f}, train acc={tacc:.4f}')
print(f'............ val loss={vloss:.4f}, val acc={vacc:.4f}')
#Final predictions
y_test = np.zeros(len(test_data), dtype=np.int32)
y_pred = np.zeros(len(test_data), dtype=np.int32)
with pt.no_grad():
for i, (data,labels) in enumerate(test_loader):
data, labels = data.to(gpu), labels.to(gpu)
samples = len(data)
output = model(data)
preds = model.predict(output).flatten()
#Copy the data from gpu memory to cpu before passing to numpy
preds_numpy = preds.cpu().numpy()
y_pred[i*samples:(i*samples)+samples] = preds_numpy
y_test[i*samples:(i*samples)+samples] = labels.cpu().numpy()
#plot loss curves and missed predictions
x_test = test_data.data
model_history = [tloss_history, vloss_history, tacc_history, vacc_history]
plot_loss_wrong_preds(model_history, x_test, y_test, y_pred, class_labels)
#save metrics
file_stream = open('results/CIFAR10.txt', 'w')
print("Model Summary\n----------\n", file=file_stream)
print(summary.summary(model, pt.zeros((1, 3, 32, 32), device=gpu)), file=file_stream)
#print_model_metrics(y_train, y_pred_train, class_labels, 'Train Metrics', file_stream)
print_model_metrics(y_test, y_pred, class_labels, 'Test Metrics', file_stream)
file_stream.close()
print_model_metrics(y_test, y_pred, class_labels, 'Test Metrics')
def init_weight(self):
for ly in self.children():
if isinstance(ly, nn.Conv2d):
nn.init.kaiming_normal_(ly.weight, a=1)
if not ly.bias is None: nn.init.constant_(ly.bias, 0)
def get_params(self):
wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params = [], [], [], []
for name, child in self.named_children():
child_wd_params, child_nowd_params = child.get_params()
if isinstance(child, (FeatureFusionModel, BiseNetOutput)):
lr_mul_wd_params += child_wd_params
lr_mul_nowd_params += child_nowd_params
else:
wd_params += child_wd_params
nowd_params += child_nowd_params
return wd_params, nowd_params, lr_mul_wd_params, lr_mul_nowd_params
if __name__ == "__main__":
net = BiSeNetV1(3)
net.cuda()
net.eval()
in_ten = torch.randn(16, 3, 640, 480).cuda()
#out, out16, out32 = net(in_ten)
#print(out.shape)
#print(out16.shape)
#print(out32.shape)
print(summary(net, in_ten, show_input=True))
net.get_params()
def train():
if args.dataset == 'COCO':
cfg = coco
dataset = COCODetection(root=args.dataset_root,
transform=SSDAugmentation(
cfg['min_dim'], MEANS))
elif args.dataset == 'VOC':
cfg = voc
dataset = VOCDetection(root=args.dataset_root,
transform=SSDAugmentation(
cfg['min_dim'], MEANS))
if args.visdom:
import visdom
global viz
viz = visdom.Visdom()
ssd_net = build_ssd('train', cfg['min_dim'], cfg['num_classes'])
net = ssd_net
# net_fp32 = ssd_net
# net_fp32.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
# net_fp32_fused = net_fp32#torch.quantization.fuse_modules(net_fp32, [['conv', 'relu']])
# net_fp32_prepared = torch.quantization.prepare(net_fp32_fused)
# input_fp32 = torch.randn(16, 3, 300, 300)
# net_fp32_prepared(input_fp32)
# net_int8 = torch.quantization.convert(net_fp32_prepared)
# result = net_int8(input_fp32)
# net = net_int8
# Display all blocks and layers of the network
if args.verbosity > 1:
print()
print(
"The network has been built successfully, the structure is shown below."
)
# print(net)
# print()
print(
summary(net,
torch.zeros((16, 3, 300, 300)),
batch_size=args.batch_size,
show_input=True,
show_hierarchical=True))
if args.cuda:
net = torch.nn.DataParallel(ssd_net)
cudnn.benchmark = True
load_quant = False
if load_quant:
net_fp32 = ssd_net
net_fp32.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
net_fp32_fused = torch.quantization.fuse_modules(
net_fp32,
[['vgg.0', 'vgg.1'], ['vgg.2', 'vgg.3'], ['vgg.5', 'vgg.6'],
['vgg.7', 'vgg.8'], ['vgg.10', 'vgg.11'], ['vgg.12', 'vgg.13'],
['vgg.14', 'vgg.15'], ['vgg.17', 'vgg.18'], ['vgg.19', 'vgg.20'],
['vgg.21', 'vgg.22'], ['vgg.24', 'vgg.25'], ['vgg.26', 'vgg.27'],
['vgg.28', 'vgg.29'], ['vgg.31', 'vgg.32'], ['vgg.33', 'vgg.34']])
net_fp32_prepared = torch.quantization.prepare_qat(net_fp32_fused)
net = net_fp32_prepared
net.eval()
net = torch.quantization.convert(net)
print('Resuming quantized model training, loading {}...'.format(
args.resume))
net.load_weights(args.resume)
net.train()
else:
if args.resume:
print('Resuming training, loading {}...'.format(args.resume))
ssd_net.load_weights(args.resume)
else:
vgg_weights = torch.load(args.save_folder + args.basenet)
print('Loading base network...')
ssd_net.vgg.load_state_dict(vgg_weights)
if args.cuda:
net = net.cuda()
if not args.resume:
print('Initializing weights...')
# initialize newly added layers' weights with xavier method
ssd_net.extras.apply(weights_init)
ssd_net.loc.apply(weights_init)
ssd_net.conf.apply(weights_init)
# quant_model = ssd_net
# quant_model.fuse_model()
# quant_model.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
# torch.quantization.prepare_qat(quant_model, inplace=True)
net_fp32 = ssd_net
net_fp32.qconfig = torch.quantization.get_default_qat_qconfig('fbgemm')
net_fp32_fused = torch.quantization.fuse_modules(
net_fp32,
[['vgg.0', 'vgg.1'], ['vgg.2', 'vgg.3'], ['vgg.5', 'vgg.6'],
['vgg.7', 'vgg.8'], ['vgg.10', 'vgg.11'], ['vgg.12', 'vgg.13'],
['vgg.14', 'vgg.15'], ['vgg.17', 'vgg.18'], ['vgg.19', 'vgg.20'],
['vgg.21', 'vgg.22'], ['vgg.24', 'vgg.25'], ['vgg.26', 'vgg.27'],
['vgg.28', 'vgg.29'], ['vgg.31', 'vgg.32'], ['vgg.33', 'vgg.34']])
net_fp32_prepared = torch.quantization.prepare_qat(net_fp32_fused)
net = net_fp32_prepared
print('Quantized Model Prepared')
print("net = ", net)
# input_fp32 = torch.randn(16, 3, 300, 300)
# net_fp32_prepared(input_fp32)
# net_int8 = torch.quantization.convert(net_fp32_prepared)
# result = net_int8(input_fp32)
# net = net_int8
# Print out all of the things that we can prune
# print(net.state_dict().keys())
parameters_to_prune = (
(net.vgg[0], 'weight'),
(net.vgg[2], 'weight'),
(net.vgg[5], 'weight'),
(net.vgg[7], 'weight'),
(net.vgg[10], 'weight'),
(net.vgg[12], 'weight'),
(net.vgg[14], 'weight'),
(net.vgg[17], 'weight'),
(net.vgg[19], 'weight'),
(net.vgg[21], 'weight'),
(net.vgg[24], 'weight'),
(net.vgg[26], 'weight'),
(net.vgg[28], 'weight'),
(net.vgg[31], 'weight'),
(net.vgg[33], 'weight'),
(net.extras[0], 'weight'),
(net.extras[1], 'weight'),
(net.extras[2], 'weight'),
(net.extras[3], 'weight'),
(net.extras[4], 'weight'),
(net.extras[5], 'weight'),
(net.extras[6], 'weight'),
(net.extras[7], 'weight'),
(net.loc[0], 'weight'),
(net.loc[1], 'weight'),
(net.loc[2], 'weight'),
(net.loc[3], 'weight'),
(net.loc[4], 'weight'),
(net.loc[5], 'weight'),
(net.conf[0], 'weight'),
(net.conf[1], 'weight'),
(net.conf[2], 'weight'),
(net.conf[3], 'weight'),
(net.conf[4], 'weight'),
(net.conf[5], 'weight'),
)
prune.global_unstructured(
parameters_to_prune,
pruning_method=prune.L1Unstructured,
amount=args.prune,
)
stored = 0
stored2 = 0
for x in [0, 2, 5, 7, 10, 12, 14, 17, 19, 21, 24, 26, 28, 31, 33]:
print(
"Sparsity in module.vgg[", x, "].weight: {:.2f}%".format(
100. * float(torch.sum(net.vgg[x].weight == 0)) /
float(net.vgg[x].weight.nelement())))
stored += torch.sum(
net.vgg[x].weight ==
0) # get the number of weights in this layer that are set to 0
stored2 += net.vgg[x].weight.nelement(
) # get the total number of elements in this layer
# if x ==0:
# print(list(net.module.vgg[x].named_parameters()))
# print(list(net.module.vgg[x].named_buffers()))
#prune.remove(net.module.vgg[x], 'weight') # make the elements pruned in this layer permanent
# if x ==0:
# print("weights have been permanently removed")
# print(list(net.module.vgg[x].named_parameters()))
# print(list(net.module.vgg[x].named_buffers()))
for x in range(8):
print(
"Sparsity in module.extras[", x, "].weight: {:.2f}%".format(
100. * float(torch.sum(net.extras[x].weight == 0)) /
float(net.extras[x].weight.nelement())))
stored += torch.sum(net.extras[x].weight == 0)
stored2 += net.extras[x].weight.nelement()
#prune.remove(net.module.extras[x], 'weight')
for x in range(6):
print(
"Sparsity in module.loc[", x, "].weight: {:.2f}%".format(
100. * float(torch.sum(net.loc[x].weight == 0)) /
float(net.loc[x].weight.nelement())))
stored += torch.sum(net.loc[x].weight == 0)
stored2 += net.loc[x].weight.nelement()
#prune.remove(net.module.loc[x], 'weight')
for x in range(6):
print(
"Sparsity in module.conf[", x, "].weight: {:.2f}%".format(
100. * float(torch.sum(net.conf[x].weight == 0)) /
float(net.conf[x].weight.nelement())))
stored += torch.sum(net.conf[x].weight == 0)
stored2 += net.conf[x].weight.nelement()
#prune.remove(net.module.conf[x], 'weight')
print("Pruned Parameters:", stored.item(), "/", stored2)
print("Global sparsity: {:.2f}%".format(100. * float(stored) /
float(stored2)))
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
criterion = MultiBoxLoss(cfg['num_classes'], 0.5, True, 0, True, 3, 0.5,
False, args.cuda)
net.train()
# loss counters
loc_loss = 0
conf_loss = 0
epoch = 0
print('Loading the dataset...')
epoch_size = len(dataset) // args.batch_size
print('Training SSD on:', dataset.name)
print('Using the specified args:')
print(args)
print()
trainStartTime = time.time()
step_index = 0
# Adjust the current learning rate based on the starting iteration
for k in range(len(cfg['lr_steps'])):
if args.start_iter > cfg['lr_steps'][k]:
print(
"Adjusting learning rate based on nonzero starting iteration.")
step_index = step_index + 1
adjust_learning_rate(optimizer, args.gamma, step_index)
# If using visdom, prepare the plot
if args.visdom:
vis_title = 'SSD (' + str(
args.prune * 100) + '% Global Pruning) on ' + dataset.name
vis_legend = ['Loc Loss', 'Conf Loss', 'Total Loss']
iter_plot = create_iter_plot('Iteration', 'Loss', vis_title,
vis_legend)
epoch_plot = create_epoch_plot('Epoch', 'Loss', vis_title, vis_legend)
data_loader = data.DataLoader(dataset,
args.batch_size,
num_workers=args.num_workers,
shuffle=True,
collate_fn=detection_collate,
pin_memory=True)
# create batch iterator
batch_iterator = iter(data_loader)
for iteration in range(args.start_iter, cfg['max_iter']):
if args.visdom and iteration != 0 and (iteration % epoch_size == 0):
epoch = epoch + 1
update_vis_plot(epoch, loc_loss, conf_loss, epoch_plot, None,
'append', epoch_size)
# reset epoch loss counters
loc_loss = 0
conf_loss = 0
# If this is an iteration where the learning rate should change
if iteration in cfg['lr_steps']:
step_index = step_index + 1
adjust_learning_rate(optimizer, args.gamma, step_index)
# load train data
try:
images, targets = next(batch_iterator)
except StopIteration:
batch_iterator = iter(data_loader)
images, targets = next(batch_iterator)
if args.cuda:
images = Variable(images.cuda())
targets = [Variable(ann.cuda(), volatile=False) for ann in targets]
else:
images = Variable(images)
targets = [Variable(ann, volatile=False) for ann in targets]
# forward
t0 = time.time() # start a timer
#print("images shape = ", images.shape)
out = net(images) # run images through the network
# backprop
optimizer.zero_grad() # reset gradients to zero
loss_l, loss_c = criterion(out, targets) # compute both types of loss
loss = loss_l + loss_c # sum both types of loss to get total loss
loss.backward(
) # compute dloss/dx for every parameter x which has requires_grad=True
optimizer.step(
) # Perform a parameter update based on the current gradient
t1 = time.time() # Get stop time
loc_loss = loc_loss + loss_l.data
conf_loss = conf_loss + loss_c.data
if iteration % 10 == 0:
print('Iteration ' + repr(iteration) + ' || Loss: %.4f ||' %
(loss.data) + ' Timer: %.4f sec.' % (t1 - t0))
if args.visdom:
update_vis_plot(iteration, loss_l.data, loss_c.data, iter_plot,
epoch_plot, 'append')
if iteration - args.start_iter != 0 and iteration % args.save_freq == 0:
saveName = 'weights/ssd300_'
saveName += (args.dataset + '_' + repr(iteration) + '_quant_' +
str(int(args.prune * 100)) + '.pth')
print('\nSaving state at iteration', iteration, 'as:', saveName)
trainTime = time.time() - trainStartTime
if trainTime < 3600:
print('Training Time: %.2f minutes' % (trainTime / 60), '\n')
else:
print('Training Time: %.2f hours' % (trainTime / 3600), '\n')
#net.eval()
# net_int8 = torch.quantization.convert(ssd_net)
# net2 = net_int8
if args.prune > 0:
for x in [
0, 2, 5, 7, 10, 12, 14, 17, 19, 21, 24, 26, 28, 31, 33
]:
prune.remove(
net.vgg[x], 'weight'
) # make the elements pruned in this layer permanent
for x in range(8):
prune.remove(net.extras[x], 'weight')
for x in range(6):
prune.remove(net.loc[x], 'weight')
for x in range(6):
prune.remove(net.conf[x], 'weight')
torch.save(net.state_dict(), saveName)
net.eval()
#net = net.to('cpu')
net_int8 = torch.quantization.convert(net)
#print(net_int8)
torch.save(net_int8.state_dict(),
'weights/quantized.pth') # Seems no smaller in size
###################################################################################################################################
#DO EVAL HERE ON net_int8????
dataset_mean = (104, 117, 123)
dataset = VOCDetection(args.dataset_root, [('2007', 'test')],
BaseTransform(300, dataset_mean),
VOCAnnotationTransform())
net_int8.phase = 'test'
net_int8.softmax = nn.Softmax(dim=-1)
net_int8.detect = Detect()
net_int8.to('cpu')
net_int8.eval()
print("Now running eval on trained, quantized network")
test_net('eval/quant/',
net_int8,
False,
dataset,
BaseTransform(net_int8.size, dataset_mean),
5,
300,
thresh=0.01)
###################################################################################################################################
# net_int8 = torch.quantization.convert(net_fp32_prepared)
print("Just finalized the weights after pruning, stopping here..")
print("Continue with higher args.prune for iterative pruning")
return
# You reached the maximum iteration, save before ending
torch.save(ssd_net.state_dict(),
args.save_folder + '' + args.dataset + '.pth')
print(input)
# print(input.shape)
# changing dimensions to suit my network
input = input.permute(0, 2, 1)
output = self.conv1(input)
# print(output.size())
output = self.conv2(output)
# print(output.size())
output = self.conv3(output)
# print(output.size())
output = self.conv4(output)
output = self.conv5(output)
output = self.conv6(output)
output = output.view(output.size(0), -1)
output = self.fc1(output)
output = self.fc2(output)
output = self.fc3(output)
# output = self.softmax(output)
# print("final output:", output)
return output
if __name__ == "__main__":
model = CharCNN(n_classes=24)
print(
summary(model,
torch.ones((1, 1024, 2)),
show_input=False,
show_hierarchical=False))
args.model.num_classes = num_classes
#############################################MODEL-DEFINITION#######################
model = model_zoo[args.model.type](**args.model)
# alternatively one can define a model directly as follows
# ```
# model = ResNet18(num_classes=num_classes, variant=args.dataset_name)
# .to(args.device)
# ```
print(
summary(
model,
torch.zeros([1] + list(data_shape)),
show_input=True,
show_hierarchical=True,
))
model = model.to(args.device)
if args.num_gpus_to_use > 1:
model = nn.parallel.DataParallel(model)
#############################################OPTIMISATION###########################
params = model.parameters()
criterion = nn.CrossEntropyLoss()
if args.optim.lower() == "sgd":
optimizer = optim.SGD(
def _document_models(self) -> None:
"""Add model summaries to the traceability document.
"""
with self.doc.create(Section("Models")):
for model in humansorted(self.system.network.models,
key=lambda m: m.model_name):
if not isinstance(model, (tf.keras.Model, torch.nn.Module)):
continue
self.doc.append(NoEscape(r'\FloatBarrier'))
with self.doc.create(Subsection(f"{model.model_name}")):
if isinstance(model, tf.keras.Model):
# Text Summary
summary = []
model.summary(line_length=92,
print_fn=lambda x: summary.append(x))
summary = "\n".join(summary)
self.doc.append(Verbatim(summary))
with self.doc.create(Center()):
self.doc.append(
HrefFEID(FEID(id(model)), model.model_name))
# Visual Summary
# noinspection PyBroadException
try:
file_path = os.path.join(
self.figure_dir,
f"FE_Model_{model.model_name}.pdf")
tf.keras.utils.plot_model(model,
to_file=file_path,
show_shapes=True,
expand_nested=True)
# TODO - cap output image size like in the pytorch implementation in case of huge network
# TODO - save raw .dot file in case system lacks graphviz
except Exception:
file_path = None
print(
f"FastEstimator-Warn: Model {model.model_name} could not be visualized by Traceability"
)
elif isinstance(model, torch.nn.Module):
if hasattr(model, 'fe_input_spec'):
# Text Summary
# noinspection PyUnresolvedReferences
inputs = model.fe_input_spec.get_dummy_input()
self.doc.append(
Verbatim(pms.summary(model, inputs)))
with self.doc.create(Center()):
self.doc.append(
HrefFEID(FEID(id(model)),
model.model_name))
# Visual Summary
# Import has to be done while matplotlib is using the Agg backend
old_backend = matplotlib.get_backend() or 'Agg'
matplotlib.use('Agg')
# noinspection PyBroadException
try:
# Fake the IPython import when user isn't running from Jupyter
sys.modules.setdefault('IPython', MagicMock())
sys.modules.setdefault('IPython.display',
MagicMock())
import hiddenlayer as hl
with Suppressor():
graph = hl.build_graph(model, inputs)
graph = graph.build_dot()
graph.attr(
rankdir='TB'
) # Switch it to Top-to-Bottom instead of Left-to-Right
graph.attr(
size="200,200"
) # LaTeX \maxdim is around 575cm (226 inches)
graph.attr(margin='0')
# TODO - save raw .dot file in case system lacks graphviz
file_path = graph.render(
filename=f"FE_Model_{model.model_name}",
directory=self.figure_dir,
format='pdf',
cleanup=True)
except Exception:
file_path = None
print(
"FastEstimator-Warn: Model {} could not be visualized by Traceability"
.format(model.model_name))
finally:
matplotlib.use(old_backend)
else:
self.doc.append(
"This model was not used by the Network during training."
)
if file_path:
with self.doc.create(Figure(position='ht!')) as fig:
fig.append(
Label(
Marker(name=str(FEID(id(model))),
prefix="model")))
fig.add_image(
os.path.relpath(file_path,
start=self.save_dir),
width=NoEscape(
r'1.0\textwidth,height=0.95\textheight,keepaspectratio'
))
fig.add_caption(
NoEscape(
HrefFEID(FEID(id(model)),
model.model_name).dumps()))
self.conv2 = nn.Conv2d(kernels_1, kernels_2, kernel_size=5, stride=(2, 2))
self.fc1 = nn.Linear(16 * kernels_2, hidden)
self.fc2 = nn.Linear(hidden, 10)
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = x.view(-1, 16 * self.kernels_2)
x = F.relu(self.fc1(x))
x = F.softmax(self.fc2(x), dim=1)
return x
# %%
net = ConvNet()
print(summary(net, torch.zeros((1, 1, 28, 28)), show_input=True))
print(summary(net, torch.zeros((1, 1, 28, 28)), show_input=False))
# %%
net2 = ConvNet(kernels_1=25, kernels_2=40, hidden=128)
print(summary(net2, torch.zeros((1, 1, 28, 28)), show_input=True))
print(summary(net2, torch.zeros((1, 1, 28, 28)), show_input=False))
# %%
conv_1_clf = NeuralNetClassifier(
ConvNet, max_epochs=20, lr=0.1, iterator_train__shuffle=True, device=device
)
# %%
for idx in range(len(name_list)):
file_name = name_list[idx]
layer_name = layer_list[idx]
weights_data = np.asarray(model.state_dict()[layer_name].tolist())
dim = len(weights_data.shape)
if dim == 3:
with open(file_name, "w+") as outfile:
outfile.write("# Matrix shape: {}\n".format(
weights_data.shape))
for data_slice in weights_data:
np.savetxt(outfile, data_slice, fmt='%-15.8f')
outfile.write("# New slice\n")
elif dim == 2 or dim == 1:
with open(file_name, "w+") as outfile:
outfile.write("# Matrix shape: {}\n".format(
weights_data.shape))
np.savetxt(outfile, weights_data, fmt='%-10.8f')
model_summary_file = "model_summary.txt"
with open(model_summary_file, "w") as msf:
print(summary(model, torch.zeros(1, NUM_MFCC, NUM_FRAMES),
show_input=True),
file=msf)
print(summary(model,
torch.zeros(1, NUM_MFCC, NUM_FRAMES),
show_input=False,
show_hierarchical=True),
file=msf)
# print(summary(model, torch.zeros(1, NUM_MFCC, NUM_FRAMES), show_input=True, show_hierarchical=True))
# %%
mlp_model = nn.Sequential(
nn.Flatten(),
nn.Linear(28 * 28, 128),
nn.ReLU(),
nn.Linear(128, 64),
nn.ReLU(),
nn.Linear(64, 10),
nn.LogSoftmax(dim=1),
).to(device)
# %%
mlp_model(torch.zeros((1, 1, 28, 28)).to(device))
# %%
print(summary(mlp_model, torch.zeros((1, 1, 28, 28)).to(device), show_input=True))
print(summary(mlp_model, torch.zeros((1, 1, 28, 28)).to(device), show_input=False))
# %%
mlp_learner = Learner(
mnist_dls,
mlp_model,
metrics=[accuracy, Precision(average="macro"), Recall(average="macro")],
)
# %%
mlp_learner.lr_find()
# %%
with mlp_learner.no_bar():
mlp_learner.fit(n_epoch=4)
for IDs in df.TITLE.values.tolist():
#パディングサイズを更新
if(len(IDs.split())>PADDING_IDX):
PADDING_IDX=len(IDs.split())
OUTPUT_SIZE = 4
# モデルの定義
my_model=my_CNN(OUTPUT_SIZE,VOCAB_SIZE,PADDING_IDX)
X = Dataset_train[1][0]
#X=torch.tensor([[250,684,0,1135,6,1599,1136,0],[42,0,38,0,40,640,1484,1]])
X2=X.unsqueeze(0)
print(f"出力:{my_model(X2)}")
print(summary(my_model,X2))
model_state=my_model.state_dict()
for layer_name in model_state:
layer_weight=my_model.state_dict()[layer_name]
print(f"[{layer_name}]:{layer_weight.size()}")
# In[ ]:
model_part=opts.model_part,
model_num=opts.model_num,
transform=transforms.Compose([ToTensor()]))
print(f'validation datagen loaded in {time.time() - start_time} seconds')
# GET MODEL
start_time = time.time()
settings.init(
opts) # add opts to global variables {required before calling FullModel
model = SedanionModelScaled()
model_time = time.time() - start_time
if opts.use_time_slot:
x_in = (torch.rand(1, 108, 495, 436), torch.rand(1, 8, 495, 436))
else:
x_in = (torch.rand(1, 108, 495, 436), torch.rand(1, 7, 495, 436))
_ = summary(model, x_in, print_summary=True)
del x_in
print(f'model created in {model_time} seconds')
opt_type = opts.optimizer
lr = opts.lr
momentum = opts.momentum
beta_1 = opts.beta_1
beta_2 = opts.beta_2
epsilon = opts.epsilon
criterion = criterion_dict['mse'] # nn.MSELoss()
metric_mae = criterion_dict['mae'] # nn.L1Loss()
other_args = {}
if opt_type == 'sgd':
