def get_dataloaders(args):
image_transform = transforms.Compose([
transforms.Resize((args.height, args.width)),
transforms.ToTensor(),
lambda x: x if x.shape[0] == 3 else x.repeat(3, 1, 1),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
mask_transform = transforms.Compose([
transforms.Resize((args.height, args.width)),
transforms.ToTensor(),
lambda m: torch.where(m > 0, torch.ones_like(m), torch.zeros_like(m)),
lambda m: F.one_hot(m[0].to(torch.int64), 2).permute(2, 0, 1).to(
torch.float32),
])
dl_args = {
'data_base_path': args.data_base_path,
'datasets': args.datasets.split(' '),
'image_transform': image_transform,
'mask_transform': mask_transform,
'batch_size': args.batch_size,
}
dl_train = get_dataloader(**dl_args, partition='train', shuffle=True)
dl_validation = get_dataloader(**dl_args,
partition='validation',
shuffle=False)
dl_test = get_dataloader(**dl_args, partition='test', shuffle=False)
dls = {
'train': dl_train,
'validation': dl_validation,
'test': dl_test,
}
return dls
def main(args):
seed = 2020
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
# Make logdir
if not os.path.exists(args.checkpoints_dir):
os.makedirs(args.checkpoints_dir)
# Load dataset
train_dataloader = get_dataloader('train', args.bs, True, args.nw)
val_dataloader = get_dataloader('val', args.bs, False, args.nw)
# Model
model = SimpleModel()
optimizer = torch.optim.SGD(
model.parameters(), lr=args.lr, momentum=0.9,
weight_decay=args.wd)
model.cuda()
train(args, train_dataloader, val_dataloader,
model, optimizer)
def main():
args = get_parser().parse_args()
print(args)
# Make datasets
train_dir = os.path.join(args.datadir, 'train')
val_dir = os.path.join(args.datadir, 'val')
print('loading train dataset')
train_loader = get_dataloader(train_dir, args.batch_size, args.pretrained,
args.augmented)
print('loading val dataset')
val_loader = get_dataloader(val_dir, args.batch_size, args.pretrained,
False)
args.num_class = 2 # np.unique(train_loader[1])
args.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Initialisation model
model = Classifier(args=args)
while model.counter['epochs'] < args.epochs:
train(model=model, dataloader=train_loader)
val(model=model, dataloader=val_loader)
if model.early_stopping.early_stop:
break
if model.writer:
model.writer.close()
def main():
# Prepare arguments
opt = get_arguments().parse_args()
if opt.dataset == "mnist" or opt.dataset == "cifar10":
opt.num_classes = 10
elif opt.dataset == "gtsrb":
opt.num_classes = 43
else:
raise Exception("Invalid Dataset")
if opt.dataset == "cifar10":
opt.input_height = 32
opt.input_width = 32
opt.input_channel = 3
elif opt.dataset == "gtsrb":
opt.input_height = 32
opt.input_width = 32
opt.input_channel = 3
elif opt.dataset == "mnist":
opt.input_height = 28
opt.input_width = 28
opt.input_channel = 1
else:
raise Exception("Invalid Dataset")
# Load models and masks
if opt.dataset == "cifar10":
netC = PreActResNet18().to(opt.device)
elif opt.dataset == "gtsrb":
netC = PreActResNet18(num_classes=43).to(opt.device)
elif opt.dataset == "mnist":
netC = NetC_MNIST().to(opt.device)
else:
raise Exception("Invalid dataset")
path_model = os.path.join(
opt.checkpoints, opt.dataset, opt.attack_mode, "{}_{}_ckpt.pth.tar".format(opt.attack_mode, opt.dataset)
)
state_dict = torch.load(path_model)
print("load C")
netC.load_state_dict(state_dict["netC"])
netC.to(opt.device)
netC.eval()
netC.requires_grad_(False)
print("load G")
netG = Generator(opt)
netG.load_state_dict(state_dict["netG"])
netG.to(opt.device)
netG.eval()
netG.requires_grad_(False)
print("load M")
netM = Generator(opt, out_channels=1)
netM.load_state_dict(state_dict["netM"])
netM.to(opt.device)
netM.eval()
netM.requires_grad_(False)
# Prepare dataloader
test_dl = get_dataloader(opt, train=False)
test_dl2 = get_dataloader(opt, train=False)
eval(netC, netG, netM, test_dl, test_dl2, opt)
def load_data(self, modes=["train", "valid", "test"]):
self.train_dataloader = get_dataloader(
self.config, mode="train") if "train" in modes else None
self.valid_dataloader = get_dataloader(
self.config, mode="valid") if "valid" in modes else None
self.test_dataloader = get_dataloader(
self.config, mode="test") if "test" in modes else None
def __init__(self):
self.experiment_id = "PPF-FoldNet" + time.strftime('%m%d%H%M')
snapshot_root = 'snapshot/%s' % self.experiment_id
tensorboard_root = 'tensorboard/%s' % self.experiment_id
os.makedirs(snapshot_root, exist_ok=True)
os.makedirs(tensorboard_root, exist_ok=True)
shutil.copy2(os.path.join('.', 'train.py'), os.path.join(snapshot_root, 'train.py'))
shutil.copy2(os.path.join('.', 'models/model_conv1d.py'), os.path.join(snapshot_root, 'model.py'))
self.epoch = 100
self.num_patches = 1
self.num_points_per_patch = 1024 # num of points per patches
self.batch_size = 32
self.dataset = 'sun3d'
self.data_train_dir = './data/3DMatch/rgbd_fragments'
self.data_test_dir = './data/3DMatch/rgbd_fragments'
self.gpu_mode = True
self.verbose = True
# model & optimizer
self.model = PPFFoldNet(self.num_patches, self.num_points_per_patch)
self.pretrain = ''
self.parameter = self.model.get_parameter()
self.optimizer = optim.Adam(self.parameter, lr=0.001, betas=(0.9, 0.999), weight_decay=1e-6)
self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.5)
self.scheduler_interval = 10
# dataloader
self.train_loader = get_dataloader(root=self.data_train_dir,
batch_size=self.batch_size,
split='train',
num_patches=self.num_patches,
num_points_per_patch=self.num_points_per_patch,
shuffle=True,
num_workers=4,
)
self.test_loader = get_dataloader(root=self.data_test_dir,
batch_size=self.batch_size,
split='test',
num_patches=self.num_patches,
num_points_per_patch=self.num_points_per_patch,
shuffle=False,
num_workers=4,
)
print("Training set size:", self.train_loader.dataset.__len__())
print("Test set size:", self.test_loader.dataset.__len__())
# snapshot
self.snapshot_interval = 100000
self.save_dir = os.path.join(snapshot_root, 'models/')
self.result_dir = os.path.join(snapshot_root, 'results/')
self.tboard_dir = tensorboard_root
# evaluate
self.evaluate_interval = 2
self.evaluate_metric = ChamferLoss()
self.check_args()
def main():
global args, best_acc1, device
criterion = PrototypicalLoss().to(device)
cudnn.benchmark = True
runs_path = glob('runs/*')
max_len_exp = max([len(x) for x in runs_path]) + 2
print(f"|{'Experiment':^{max_len_exp}}|{'Loss':^17}|{'ACC':^17}|")
except_list = []
pl_mi = u"\u00B1"
for exp in glob('runs/*'):
checkpoint, args = None, None
files = glob(exp + '/*')
for file in files:
if file.endswith('model_best.pth'):
checkpoint = torch.load(os.path.abspath(file))
elif file.endswith('.json'):
params = json.load(open(os.path.abspath(file)))
args = SimpleNamespace(**params)
if checkpoint is None or args is None:
except_list.append(f"checkpoint and params are not exist in {exp}")
continue
if args.dataset == 'omniglot':
test_loader = get_dataloader(args, 'test')
else:
test_loader = get_dataloader(args, 'val')
input_dim = 1 if args.dataset == 'omniglot' else 3
if args.model == 'protonet':
model = ProtoNet(input_dim).to(device)
else:
model = ResNet(input_dim).to(device)
model.load_state_dict(checkpoint['model_state_dict'])
best_acc1 = checkpoint['best_acc1']
loss_list, acc_list = test(test_loader, model, criterion)
loss, loss_moe = margin_of_error(loss_list)
acc, acc_moe = margin_of_error(acc_list)
loss_string = f'{loss:.3f} {pl_mi} {loss_moe:.3f}'
acc_string = f'{acc:.3f} {pl_mi} {acc_moe:.3f}'
print(f"|{exp:^{max_len_exp}}|{loss_string:^16}|{acc_string:^16}|")
if len(except_list):
pp(except_list)
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs):
ckpt_dir = os.path.join(ckpt_path, '')
ckpt_dir_flag = os.path.join(ckpt_dir, flag)
prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path))
train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl')
dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl')
os.makedirs(ckpt_dir, exist_ok=True)
with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f:
f.write('ckpt_dir')
os.makedirs(ckpt_dir_flag, exist_ok=True)
with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f:
f.write('ckpt_dir_flag')
file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log'))
logging.getLogger().addHandler(file_handler)
if os.path.exists(train_pkl):
logger.info(f'=> load train, dev data from {ckpt_dir}')
train_data = pk.load(open(train_pkl, 'rb'))
# train_set is a of 1600 elements which are dict.
# with keys:'mel' for mel-spectogram np array (80,xxx) and 'speaker' for speaker string ('p225')
# print(f"**********************data is: {train_data[0]}********************")
# print(f"**********************type is: {train_data[0]['mel'].shape}********************")
dev_data = pk.load(open(dev_pkl, 'rb'))
train_set = get_dataset(dset='train',
dataset_config=dataset_config,
njobs=njobs,
metadata=train_data)
# train_set is a class: dataloader.vctk.Dataset.
# Has attribute data such that train_set.data is a list of 1600 features so train_set.data[i] is o size (80,xxx)
# print(f"**********************type is: {type(train_set)}********************")
# print(f"**********************shape is: {train_set.data[0]['mel'].shape}********************")
dev_set = get_dataset(dset='dev',
dataset_config=dataset_config,
njobs=njobs,
metadata=dev_data)
else:
train_set = get_dataset(dset='train',
dataset_config=dataset_config,
njobs=njobs)
dev_set = get_dataset(dset='dev',
dataset_config=dataset_config,
njobs=njobs)
pk.dump(train_set.data, open(train_pkl, 'wb'))
pk.dump(dev_set.data, open(dev_pkl, 'wb'))
train_loader = get_dataloader(dset='train',
dataloader_config=dataloader_config,
dataset=train_set)
dev_loader = get_dataloader(dset='dev',
dataloader_config=dataloader_config,
dataset=dev_set)
# print(f"**********************type is: {type(train_loader)}********************")
return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def load_data(ckpt_path, dataset_config, dataloader_config, njobs):
if os.path.isdir(ckpt_path):
d = os.path.join(ckpt_path, '')
with open(os.path.join(d, 'dirtype'), 'r') as f:
dirtype = f.read().strip()
if dirtype == 'ckpt_dir':
logger.warn(
f'The ckpt_path is {ckpt_path}, the flag is not specified.'
)
logger.warn(f'Use "default" flag.')
ckpt_dir = d
flag = 'default'
ckpt_dir_flag = os.path.join(ckpt_dir, flag)
ckpt_path = ckpt_dir_flag
elif dirtype == 'ckpt_dir_flag':
ckpt_dir_flag = os.path.dirname(d)
ckpt_dir = os.path.dirname(ckpt_dir_flag)
flag = os.path.basename(ckpt_dir_flag)
else:
raise NotImplementedError(
f'Wrong dirtype: {dirtype} from {d}.')
else:
ckpt_dir_flag = os.path.dirname(ckpt_path)
ckpt_dir = os.path.dirname(ckpt_dir_flag)
flag = os.path.basename(ckpt_dir_flag)
file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log'))
logging.getLogger().addHandler(file_handler)
logger.info(f'=> load train, dev data from {os.path.join(ckpt_dir)}')
prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path))
train_data = pk.load(
open(os.path.join(ckpt_dir, f'{prefix}_train.pkl'), 'rb'))
dev_data = pk.load(
open(os.path.join(ckpt_dir, f'{prefix}_dev.pkl'), 'rb'))
train_set = get_dataset(dset='train',
dataset_config=dataset_config,
njobs=njobs,
metadata=train_data)
dev_set = get_dataset(dset='dev',
dataset_config=dataset_config,
njobs=njobs,
metadata=dev_data)
train_loader = get_dataloader(dset='train',
dataloader_config=dataloader_config,
dataset=train_set)
dev_loader = get_dataloader(dset='dev',
dataloader_config=dataloader_config,
dataset=dev_set)
return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def __init__(self):
self.experiment_id = "FoldNet" + time.strftime('%m%d%H%M')
snapshot_root = 'snapshot/%s' % self.experiment_id
tensorboard_root = 'tensorboard/%s' % self.experiment_id
os.makedirs(snapshot_root, exist_ok=True)
os.makedirs(tensorboard_root, exist_ok=True)
shutil.copy2(os.path.join('.', 'train.py'), os.path.join(snapshot_root, 'train.py'))
self.epoch = 300
self.num_points = 2048
self.batch_size = 16
self.dataset = 'shapenet'
self.data_dir = 'data/shapenetcore_partanno_segmentation_benchmark_v0'
self.gpu_mode = torch.cuda.is_available()
self.verbose = False
# model & optimizer
self.model = FoldNet(self.num_points)
self.pretrain = ''
self.parameter = self.model.get_parameter()
self.optimizer = optim.Adam(self.parameter, lr=0.001, betas=(0.9, 0.999), weight_decay=1e-6)
self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, gamma=0.5)
self.scheduler_interval = 100
# dataloader
self.train_loader = get_dataloader(root=self.data_dir,
split='train',
classification=True,
batch_size=self.batch_size,
num_points=self.num_points,
shuffle=True
)
self.test_loader = get_dataloader(root=self.data_dir,
split='test',
classification=True, # if True then return pts & cls
batch_size=self.batch_size,
num_points=self.num_points,
shuffle=False
)
print("Training set size:", self.train_loader.dataset.__len__())
print("Test set size:", self.test_loader.dataset.__len__())
# snapshot
self.snapshot_interval = 10
self.save_dir = os.path.join(snapshot_root, 'models/')
self.result_dir = os.path.join(snapshot_root, 'results/')
self.tboard_dir = tensorboard_root
self.check_args()
def train(opt, init_mask, init_pattern):
test_dataloader = get_dataloader(opt, train=False)
# Build regression model
regression_model = RegressionModel(opt, init_mask,
init_pattern).to(opt.device)
# Set optimizer
optimizerR = torch.optim.Adam(regression_model.parameters(),
lr=opt.lr,
betas=(0.5, 0.9))
# Set recorder (for recording best result)
recorder = Recorder(opt)
for epoch in range(opt.epoch):
early_stop = train_step(regression_model, optimizerR, test_dataloader,
recorder, epoch, opt)
if (early_stop):
break
# Save result to dir
recorder.save_result_to_dir(opt)
return recorder, opt
def evaluate(model, config, mode="test"):
# load data
dataloader = get_dataloader(config, mode=mode)
# evaluate
for i, data in enumerate(dataloader):
pass
def main_fc(basetype, train=True, cluttered_translated=False):
setup_dirs(data_dir='./data', ckpt_dir='./ckpt')
torch.manual_seed(1)
if torch.cuda.is_available():
torch.cuda.manual_seed(1)
# get dataloaders
batch_size = 20 if train else 1000
data_loader = dataloader.get_dataloader(
data_dir='./data',
batch_size=batch_size,
random_seed=1,
is_train=train,
valid_size=0.1,
shuffle=train,
pin_memory=torch.cuda.is_available(),
cluttered_translated=cluttered_translated)
trainer = BaseTrainer(basetype, train, data_loader, cluttered_translated)
if train:
trainer.train()
else:
trainer.test()
return
def load_data(C, logger):
data_train, data_test, data_valid, relations, rel_weights = get_dataloader(
C.dataset)(
C,
logger,
C.train_text_1,
C.train_rels_1,
C.train_text_2,
C.train_rels_2,
C.test_text,
C.test_rels,
C.valid_text,
C.valid_rels,
C.dataset,
C.rel_weight_smooth,
C.rel_weight_norm,
)
logger.log("num of sents / entity pairs in train: %d / %d" %
(len(data_train),
sum([len(x.ents) * (len(x.ents) - 1) / 2
for x in data_train])))
logger.log("num of sents / instances in test : %d / %d" %
(len(data_test), sum([len(x.ans) for x in data_test])))
logger.log("num of sents / instances in valid : %d / %d" %
(len(data_valid), sum([len(x.ans) for x in data_valid])))
logger.log("num of sents / entity pairs in test : %d / %d" %
(len(data_test),
sum([len(x.ents) * (len(x.ents) - 1) / 2 for x in data_test])))
logger.log("num of sents / entity pairs in valid : %d / %d" %
(len(data_valid),
sum([len(x.ents) * (len(x.ents) - 1) / 2
for x in data_valid])))
return data_train, data_test, data_valid, relations, rel_weights
def create_embeddings(config):
exper_folder = config['dir']
embed_folder = config['embed_folder']
config_file = 'experiment/' + exper_folder + '/config.yaml'
with open(config_file, 'r') as f:
config = yaml.load(f)
dataloader = get_dataloader(config, scope='test')
model = get_model(config)
model_weights = 'experiment/' + exper_folder + '/' + config['weights']
model.load_state_dict(torch.load(model_weights)['model'])
if os.path.exists(embed_folder):
shutil.rmtree(embed_folder)
os.makedirs(embed_folder)
with torch.no_grad():
for num_batch, sample in enumerate(dataloader):
print('Batch {}/{} processing...'.format(num_batch + 1,
len(dataloader)))
sample['anchor'] = sample['anchor'].cuda(0)
embeddings = model(sample['anchor'])
embeddings = embeddings.detach().cpu().numpy()
for num in range(len(sample['name'])):
name = sample['name'][num].replace('/', '-')
name = os.path.join(embed_folder, name)
vector = embeddings[num]
np.save(name, vector)
def test_process(self, best_model_file=None):
if best_model_file is not None:
self.best_model_file = best_model_file
self.load_state_dict(torch.load(self.best_model_file))
test_loader = get_dataloader(self.config,
mode="test",
n_way=self.config.n_way,
n_shot=self.config.n_shot,
n_query=self.config.n_query_test)
avg_acc = 0
avg_h = 0
best_acc = 0
best_h = 0
test_num = 10
for epoch in range(test_num):
acc, h = self.test_loop(test_loader)
self.logger.info("Test Acc = {:4.2f}+-{:4.2f}".format(acc, h))
if acc > best_acc:
best_acc = acc
best_h = h
avg_acc += acc
avg_h += h
avg_acc /= test_num
avg_h /= test_num
self.logger.info("Best Test Acc = {:4.2f}+-{:4.2f}".format(
best_acc, best_h))
self.logger.info("Avg Test Acc = {:4.2f}+-{:4.2f}".format(
avg_acc, avg_h))
def main(config) -> None:
talk_model = model.Model(config.model.vocab_size)
talk_dataloader = dataloader.get_dataloader(
config.data.data_category, config.train_conf.batch_size)
loss = nn.CrossEntropyLoss()
talk_trainer = trainer.Trainer(talk_model, talk_dataloader, loss,
config.train_conf.optimizer, config.train_conf.optimizer_lr)
talk_trainer.train(epoch=config.train_conf.epoch)
def inference_ocl_attr(model, current_epoch, current_iter, dataset_name,
prev_metric_dict, seen_objects, finished_objects,
all_objects, max_instance):
"""
:param model:
:param current_epoch:
:param current_iter:
:param prev_metric_dict: {attr_acc: <attr>: [acc1, acc2]}
:param filter_objects:
:param filter_attrs:
:return:
"""
model.train(False)
device = torch.device('cuda')
if not prev_metric_dict:
prev_metric_dict = {
'attr_acc': defaultdict(list),
'inst_num': defaultdict(list),
'ft_dict': defaultdict(list),
'forget_dict': defaultdict(list),
'obj_acc': defaultdict(list),
'length': 0
}
pbar = tqdm(total=len(all_objects), desc="Validation in progress")
# only seen objects by this time
for obj in all_objects:
dataloader = get_dataloader(model.cfg,
'val',
False,
False,
filter_obj=[obj])
obj_acc, attr_acc, inst_num = inference(model,
current_epoch,
current_iter,
0,
dataloader,
dataset_name,
max_instance=max_instance,
mute=True)
prev_metric_dict['attr_acc'][obj].append(attr_acc)
prev_metric_dict['inst_num'][obj].append(inst_num)
prev_metric_dict['obj_acc'][obj].append(obj_acc)
pbar.update(1)
metric_dict = prev_metric_dict
#run_forward_transfer_metrics(metric_dict, seen_objects, all_objects, 'attr_acc', metric_dict['ft_dict'])
run_forget_metrics(metric_dict, finished_objects, all_objects, 'attr_acc',
metric_dict['forget_dict'])
metric_dict['length'] += 1
numerical_metric_dict = numericalize_metric_scores(metric_dict)
return metric_dict, numerical_metric_dict
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--seq_embed_size', default=64)
parser.add_argument('--k_hops', default=3)
parser.add_argument('--hop_layers', default=3)
parser.add_argument('--lstm_layers', default=3)
parser.add_argument('--fn_layers', default=3)
parser.add_argument('--lr', default=1e-4)
parser.add_argument('--weight_decay', default=1e-8)
parser.add_argument('--alpha', default=0.6)
parser.add_argument('--epochs', default=50)
parser.add_argument('--batch_size', default=2)
parser.add_argument('--window_size', default=30)
parser.add_argument('--device', default='cpu')
args = parser.parse_args()
if torch.cuda.is_available():
args.device = "cuda:0"
# Load Data
logger.info("Loading data...")
nasdaq_data = pickle.load(open(NASDAQ_PICKLE_URL, "rb"))
model_tickers = nasdaq_data.columns.tolist()
# Load dataloaders & graphs
inputs, dataloaders = get_dataloader(data=nasdaq_data,
batch_size=args.batch_size,
window_size=args.window_size,
device=args.device)
N, n_features = inputs
train_loader, test_loader, val_loader = dataloaders
encoding, _ = get_encodings('20180105', 'NASDAQ', model_tickers)
relational_encoding = torch.FloatTensor(encoding).to(args.device)
print("relational_encoding", relational_encoding.size())
# Initialize Model
model = TemporalSAGE(
input_shape=[args.batch_size, args.window_size, N, n_features],
seq_embed_size=args.seq_embed_size,
relational_encoding=relational_encoding,
k_hops=args.k_hops,
hop_layers=args.hop_layers,
lstm_layers=args.lstm_layers,
fn_layers=args.fn_layers,
device=args.device)
logger.info(f"\n{model}")
logger.info(f"Starting training with args:")
for arg in vars(args):
logger.info(f"{arg}: {getattr(args, arg)}")
train(model=model,
train_loader=train_loader,
val_loader=val_loader,
args=args)
def gen_data(ckpt_path, flag, dataset_config, dataloader_config, njobs):
ckpt_dir = os.path.join(ckpt_path, '')
ckpt_dir_flag = os.path.join(ckpt_dir, flag)
prefix = os.path.basename(os.path.dirname(dataset_config.indexes_path))
train_pkl = os.path.join(ckpt_dir, f'{prefix}_train.pkl')
dev_pkl = os.path.join(ckpt_dir, f'{prefix}_dev.pkl')
os.makedirs(ckpt_dir, exist_ok=True)
with open(os.path.join(ckpt_dir, 'dirtype'), 'w') as f:
f.write('ckpt_dir')
os.makedirs(ckpt_dir_flag, exist_ok=True)
with open(os.path.join(ckpt_dir_flag, 'dirtype'), 'w') as f:
f.write('ckpt_dir_flag')
file_handler = logging.FileHandler(os.path.join(ckpt_dir, 'train.log'))
logging.getLogger().addHandler(file_handler)
if os.path.exists(train_pkl):
logger.info(f'=> load train, dev data from {ckpt_dir}')
train_data = pk.load(open(train_pkl, 'rb'))
dev_data = pk.load(open(dev_pkl, 'rb'))
train_set = get_dataset(dset='train',
dataset_config=dataset_config,
njobs=njobs,
metadata=train_data)
dev_set = get_dataset(dset='dev',
dataset_config=dataset_config,
njobs=njobs,
metadata=dev_data)
else:
train_set = get_dataset(dset='train',
dataset_config=dataset_config,
njobs=njobs)
dev_set = get_dataset(dset='dev',
dataset_config=dataset_config,
njobs=njobs)
pk.dump(train_set.data, open(train_pkl, 'wb'))
pk.dump(dev_set.data, open(dev_pkl, 'wb'))
train_loader = get_dataloader(dset='train',
dataloader_config=dataloader_config,
dataset=train_set)
dev_loader = get_dataloader(dset='dev',
dataloader_config=dataloader_config,
dataset=dev_set)
return ckpt_dir_flag, train_set, dev_set, train_loader, dev_loader
def main():
feature_shape = (28, 28, 512)
img_shape = (224, 224, 3)
datagen = dataloader.get_dataloader('data', batch_size=4, shuffle=True)
model = drn.drn_c_26(pretrained=True)
generator = LabelGenerator(img_shape, feature_shape)
result, images = generator.generate(datagen, model)
print(result.shape)
print(images.shape)
def main():
opt = get_arguments().parse_args()
# prepare model
classifier = MNIST_classifier()
# prepare optimizer
optimizer = torch.optim.SGD(classifier.parameters(), opt.lr, momentum=0.9)
# prepare scheduler
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
opt.scheduler_milestones,
opt.scheduler_lambda)
# prepare dataloader
dl_train = get_dataloader(opt, train=True)
dl_test = get_dataloader(opt, train=False)
# continue training ?
create_dir(opt.checkpoint)
path_model = os.path.join(opt.checkpoint, 'model_ckpt.pth.tar')
if (os.path.exists(path_model)):
print('Continue Training')
state_dict = torch.load(path_model)
classifier.load_state_dict(state_dict['classifier'])
optimizer.load_state_dict(state_dict['optimizer'])
scheduler.load_state_dict(state_dict['scheduler'])
best_acc = state_dict['best_acc']
epoch = state_dict['epoch']
else:
print('Train from scratch!!')
best_acc = 0.
epoch = 0
for epoch_idx in range(opt.n_iters):
print('Epoch {}:'.format(epoch))
train(classifier, optimizer, scheduler, dl_train, opt)
best_acc = evaluate(classifier, optimizer, scheduler, dl_test,
best_acc, epoch, opt)
epoch += 1
def train_process(self):
train_loader = get_dataloader(self.config,
mode="train",
n_way=self.config.n_way,
n_shot=self.config.n_shot,
n_query=self.config.n_query_train)
val_loader = get_dataloader(self.config,
mode="val",
n_way=self.config.n_way,
n_shot=self.config.n_shot,
n_query=self.config.n_query_test)
for param in self.feature_extractor.layer3.block[
0].conv_offset_mask.parameters():
param.requires_grad = False
for param in self.feature_extractor.layer4.block[
0].conv_offset_mask.parameters():
param.requires_grad = False
for epoch in range(50):
try:
if epoch >= 5:
for param in self.feature_extractor.layer3.block[
0].conv_offset_mask.parameters():
param.requires_grad = True
for param in self.feature_extractor.layer4.block[
0].conv_offset_mask.parameters():
param.requires_grad = True
self.train_loop(epoch, train_loader)
self.val_loop(epoch, val_loader)
except KeyboardInterrupt:
if self.best_model_file == None:
os.remove(self.logfile_name)
else:
print(Fore.YELLOW, end="")
self.logger.info("KeyboardInterrupt!")
self.post_process()
print(Style.RESET_ALL, end="")
exit()
def __init__(self, config):
self.cuda = int(config['cuda'])
self.train_dataloader = get_dataloader(config, scope='train')
self.val_dataloader = get_dataloader(config, scope='val')
self.model = get_model(config)
try:
model_weights = 'experiment/' + config['dir'] + '/' + config['weights']
self.model.load_state_dict(torch.load(model_weights)['model'])
print('Weigths loaded')
except:
print('Weights randomized')
self.optimizer = get_optimizer(config, self.model)
self.total_epochs = config['epochs']
self.batches_per_epoch = config['batches_per_epoch']
self.val_batches_per_epoch = config['val_batches_per_epoch']
self.exp_name = config['dir']
self.save_weights_regularity = config['save_weights_regularity']
self.final_weights_file = 'experiment/' + config['dir'] + '/weights_last.pth'
self.log_file = 'experiment/' + config['dir'] + '/logs.csv'
self.loss = get_loss(config)
def main(args):
# Load dataset
test_dataloader = get_dataloader('test', args.bs, False, args.nw)
# Model
model = SimpleModel()
model.cuda()
ckpt = torch.load(os.path.join(args.checkpoints_dir, 'last_ckpt.pth'))
model.load_state_dict(ckpt['model_state'])
result = test(args, test_dataloader, model)
# Make csv file
df = pd.DataFrame({'id': test_dataloader.dataset.ids, 'category': result})
df.to_csv('out.csv', index=False)
def show_reconstructed(model, class_choice='Airplane'):
dataroot = "data/shapenetcore_partanno_segmentation_benchmark_v0"
dataloader = get_dataloader(root=dataroot,
split='test',
class_choice=class_choice,
classification=True,
num_points=2048,
shuffle=False)
pts, _ = dataloader.dataset[random.randint(0, 100)]
reconstructed_pl = model(pts.view(1, 2048, 3))[0]
ax1, _ = draw_pts(pts, clr=None, cmap='CMRmap')
ax2, _ = draw_pts(reconstructed_pl.detach().numpy(),
clr=None,
cmap='CMRmap')
ax1.figure.show()
ax2.figure.show()
def run_topk_bottomk(
k, scoredir, logdir, datadir, dataname, version, lam, loss, scale, iteration, save_path, device
):
# load raw data
raw = pkl.load(open(os.path.join(datadir, dataname),'rb'))
# Load anomaly score
dr_score = pd.read_csv(scoredir)
_, _, testloader = get_dataloader(data_root=datadir,
data_name=dataname,
scale=scale,
batch_size=1)
# define window size and the number of features
window_size = testloader.dataset.window_size
nb_features = testloader.dataset.data.size(-1)
# loss function
if loss == 'mse':
gen_criterion = torch.nn.MSELoss()
elif loss == 'mae':
gen_criterion = torch.nn.L1Loss()
# Load saved model
# logs 안에 version이 있어야함. version은 학습시 log 정보와 함께 생성됨
G_weights, _, _, _, _, best_metrics = load_model(resume=version, logdir=logdir)
print('Best Loss: ',best_metrics)
G = LSTMGenerator(nb_features = nb_features).to(device)
G.load_state_dict(G_weights)
# Visualization
viz_topk_bottomk(k=k,
anomaly_score=dr_score,
G=G,
dataloader=testloader,
window_size=window_size,
nb_features=nb_features,
criterion=gen_criterion,
iteration=iteration,
colnames=raw.columns,
device=device,
save_path=save_path)
def evaluate_on_dataset(model, args):
model.eval()
model.cuda()
dataloader = get_dataloader(args, is_train=True)
for batch_nb, batch in enumerate(dataloader):
rgb = batch['rgb'].cuda()
skier = batch['skier'].cuda()
flags = batch['flags'].cuda()
rocks = batch['rocks'].cuda()
trees = batch['trees'].cuda()
gt_masks = torch.cat((skier, flags, rocks, trees), dim=1)
pred_masks, latent = model(rgb)
all_images = model.make_visuals(rgb, gt_masks, pred_masks)
pass
def predict(model):
criterion = nn.CrossEntropyLoss()
_, test_loader = get_dataloader()
pred_list = []
output_list = []
target_list = []
for data in test_loader:
input = data['input']
target = data['target']
output = model(input)
correct = torch.max(output.detach(), 1)[1] == target
pred_list.append(sum(correct.numpy()) / len(correct.numpy()))
output_list.append(output.detach().numpy())
target_list.append(target.detach().numpy())
print('Correct answer rate: {:.1f} %'.format(
sum(pred_list) / len(pred_list) * 100))
def main():
model = TimeSeriesModel(n_bins, n_lag).to(device)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
scheduler = lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.97)
if start_epoch != 0:
checkpoint = torch.load(
'./log/checkpoint_epoch{}.pth'.format(start_epoch - 1))
model.load_state_dict(checkpoint['state_dict'])
data_loaders, data_size = get_dataloader(batch_size=128, n_lag=n_lag)
for epoch in range(start_epoch, num_epochs + start_epoch):
print(80 * '=')
print('Epoch [{}/{}]'.format(epoch, num_epochs + start_epoch - 1))
t0 = time.time()
loss = train_valid(model, optimizer, scheduler, epoch, data_loaders,
data_size)
scheduler.step()
t1 = time.time()
#print("It took {0}s".format(t1-t0))
print(80 * '=')