def ts_loader(ts: TrainingSet, batch_size: int = 32) -> data.DataLoader:
'''Training set loader'''
class TSDataset(data.Dataset):
def __init__(self, ts: TrainingSet):
stateaction, reward, state_prime = (torch.tensor(x).float()
for x in ts)
self.stateaction, self.reward = stateaction, reward
self.stateaction_prime = torch.cat(
[ # all combinations of x' and u'
state_prime.unsqueeze(1).expand(-1, len(U), -1),
torch.tensor(U).expand(len(state_prime), -1).unsqueeze(-1)
],
dim=-1)
def __len__(self) -> int:
return len(self.stateaction)
def __getitem__(
self,
i: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return self.stateaction[i], self.reward[i], self.stateaction_prime[
i]
dataset = TSDataset(ts)
loader = data.DataLoader(dataset,
batch_size=None,
sampler=data.BatchSampler(
data.RandomSampler(dataset), batch_size,
False))
return loader
def make_loader_mt(dataset, batch_size):
"""Construct sampler that randomly chooses N items from N-sample dataset,
weighted so that it's even across all tasks (so no task implicitly has
higher priority than the others). Assumes the given dataset is a
TensorDataset produced by trajectories_to_dataset_mt."""
task_ids = dataset.tensor_dict['obs'].task_id
assert len(task_ids) > 0 and batch_size > 0, \
f"either {len(task_ids)}=0 task IDs or {batch_size}=0 batch size"
unique_ids, frequencies = torch.unique(task_ids,
return_counts=True,
sorted=True)
# all tasks must be present for this to work
assert torch.all(unique_ids == torch.arange(len(unique_ids))), (unique_ids)
freqs_total = torch.sum(frequencies).to(torch.float)
unique_weights = freqs_total / frequencies.to(torch.float)
unique_weights = unique_weights / unique_weights.sum()
weights = unique_weights[task_ids]
# even out the number of samples to be a multiple of batch size, always
n_samples = len(weights) + (-len(weights)) % batch_size
assert n_samples >= len(weights) and 0 == n_samples % batch_size, \
(batch_size, n_samples)
weighted_sampler = data.WeightedRandomSampler(weights,
n_samples,
replacement=True)
batch_sampler = data.BatchSampler(weighted_sampler,
batch_size=batch_size,
drop_last=True)
loader = data.DataLoader(dataset,
pin_memory=False,
batch_sampler=batch_sampler,
collate_fn=fixed_default_collate)
return loader
def __init__(self, args):
self.args = args
self.log_str_path = './train_log/' + args.model + str(
args.number) + args.mod + args.dataset + 'GL' + str(
args.global_lr) + 'LE' + str(args.local_epoch) + 'LL' + str(
args.local_lr) + '.log'
self.mod_str_path = './saved_models/' + args.model + str(
args.number) + args.mod + args.dataset + 'GL' + str(
args.global_lr) + 'LE' + str(args.local_epoch) + 'LL' + str(
args.local_lr)
self.log = Logging(self.log_str_path)
self.user_num, self.item_num, self.train_data, self.train_support_mat, self.val_negative_dict, self.val_negative, self.train_mat, self.sup_max, self.query_max = data_prepare.load_all(
self.args)
self.Train_data = data_prepare.Train_data(self.train_data,
self.item_num,
self.train_support_mat,
self.sup_max, self.query_max,
self.args)
self.Train_data.ng_train_sample()
eval_ = args.model + "(self.item_num,args).cuda()"
self.model = eval(eval_)
self.maml_train_batch_sampler = data.BatchSampler(
data.RandomSampler(range(self.Train_data.idx)),
batch_size=self.args.batch_size,
drop_last=False)
def val_on(self, dset):
self.enc.eval()
self.dec.eval()
self.mlp.eval()
pred_list = []
idx_list = []
dl = data.DataLoader(dset,
batch_sampler=data.BatchSampler(
data.RandomSampler(dset),
self.batch_size,
drop_last=False),
pin_memory=False)
for batch_idx, (xb, yb, idx) in enumerate(dl):
latent = self.enc(xb)
label_pred = self.mlp(latent) if latent.ndim == 2 else self.mlp(
latent[:, :, 0, 0])
pred_list.append(label_pred.argmax(axis=1).detach().cpu().numpy())
idx_list.append(idx.detach().cpu().numpy())
if ARGS.test: break
pred_array = np.concatenate(pred_list)
if ARGS.test:
return -1, -1, dummy_labels(self.num_classes, len(dset.y))
idx_array = np.concatenate(idx_list)
pred_array_ordered = np.array([
item[0]
for item in sorted(zip(pred_array, idx_array), key=lambda x: x[1])
])
if get_num_labels(dset.y) == 1: set_trace()
acc = -1 if ARGS.test else accuracy(pred_array_ordered,
dset.y.detach().cpu().numpy())
f1 = -1 if ARGS.test else mean_f1(pred_array_ordered,
dset.y.detach().cpu().numpy())
return acc, f1, pred_array_ordered
def batchPredict(self, dataset: LargePatentDataset, batchSize):
batchSampler = data.BatchSampler(data.SequentialSampler(dataset),
batch_size=batchSize, drop_last=False)
dataGenerator = data.DataLoader(dataset, batch_sampler=batchSampler,
collate_fn=dataset.collate)
with torch.no_grad():
with tqdm(total=len(dataset), unit='st', unit_scale=True, file=sys.stdout) as t:
t.set_description('Predicting')
for sentsTokens, batchGoldTags in dataGenerator:
actualBatchLen = len(sentsTokens)
# truncate sentence to maxInputLen (-2 to account for [CLS] & [SEP] tokens)
sentsTokens = [["[CLS]"] + sent[:self.maxInputLen-2] + ["[SEP]"] for sent in sentsTokens]
sentsTokenIds = [
convertTokensToIds(sent, '[UNK]', lambda token: dataset.tokenizer.vocab[token])
for sent in sentsTokens]
paddedSentsTokenIds = torch.tensor(padBatchTokenIds(sentsTokenIds, 0, len(sentsTokenIds[0])),
dtype=torch.long, device=self.device)
attentionMask = torch.tensor([[float(i > 0) for i in sent] for sent in paddedSentsTokenIds],
device=self.device)
batchLogits = self.model(input_ids=paddedSentsTokenIds, attention_mask=attentionMask).sigmoid()
batchLogits: np.ndarray = batchLogits.detach().cpu().numpy()
# print(batchLogits)
maxIndices = batchLogits.argmax(axis=1)
# print(maxIndices)
batchPredTags = self.tagDict.indices2words(maxIndices)
t.update(actualBatchLen)
yield batchPredTags, batchGoldTags
def get_dataloader(data_dir, batch_size, num_workers, input_size, mean, std,
distributed):
"""Get dataloader."""
def val_batch_fn(batch, device):
data = batch[0].to(device)
scale = batch[1]
center = batch[2]
score = batch[3]
imgid = batch[4]
return data, scale, center, score, imgid
val_dataset = COCOKeyPoints(data_dir,
aspect_ratio=4. / 3.,
splits=('person_keypoints_val2017'))
meanvec = [float(i) for i in mean.split(',')]
stdvec = [float(i) for i in std.split(',')]
transform_val = SimplePoseDefaultValTransform(
num_joints=val_dataset.num_joints,
joint_pairs=val_dataset.joint_pairs,
image_size=input_size,
mean=meanvec,
std=stdvec)
val_tmp = val_dataset.transform(transform_val)
sampler = make_data_sampler(val_tmp, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler,
batch_size=batch_size,
drop_last=False)
val_data = data.DataLoader(val_tmp,
batch_sampler=batch_sampler,
num_workers=num_workers)
return val_dataset, val_data, val_batch_fn
def __init__(self, dataset, batch_size, device='cpu'):
super().__init__()
self._dataset = dataset
self.batch_size = batch_size
self.device = device
self._sampler = data.BatchSampler(data.RandomSampler(
self._dataset,
replacement=False),
self.batch_size, False)
def get_dataloader(batch_size, num_workers, data_root, distributed):
transform_test = transforms.Compose([
transforms_cv.ToTensor(),
transforms_cv.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])
])
val_dataset = CIFAR10(root=data_root, train=False, transform=transform_test, download=True)
sampler = make_data_sampler(val_dataset, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler, batch_size=batch_size, drop_last=False)
val_loader = data.DataLoader(val_dataset, batch_sampler=batch_sampler, num_workers=num_workers)
return val_loader
def get_dataloader(val_dataset, batch_size, num_workers, distributed, coco=False):
"""Get dataloader."""
if coco:
batchify_fn = Tuple(Stack(), Pad(pad_val=-1), Empty())
else:
batchify_fn = Tuple(Stack(), Pad(pad_val=-1))
sampler = make_data_sampler(val_dataset, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler, batch_size=batch_size, drop_last=False)
val_loader = data.DataLoader(val_dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn,
num_workers=num_workers)
return val_loader
def get_dataloader(val_dataset, batch_size, num_workers, distributed, coco=False):
"""Get dataloader."""
if coco:
batchify_fn = Tuple(*[Append() for _ in range(3)], Empty())
else:
batchify_fn = Tuple(*[Append() for _ in range(3)])
sampler = make_data_sampler(val_dataset, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler, batch_size=batch_size, drop_last=False)
val_loader = data.DataLoader(val_dataset, batch_sampler=batch_sampler, collate_fn=batchify_fn,
num_workers=num_workers)
return val_loader
def batches(self, size=None):
# if self.data_loader is None:
# self.data_loader = torch_data.DataLoader(self, batch_size=size, shuffle=self.shuffle_batches)
# for sample in self.data_loader:
# yield sample
sampler_name = torch_data.RandomSampler if self.shuffle_batches else torch_data.SequentialSampler
sampler = torch_data.BatchSampler(sampler_name(range(len(self))),
size,
drop_last=False)
for batch_idx in sampler:
# let wpid_batch and wplabel_batch contain the list of ids and labels, respectively
wpid_batch = [self.wpids[idx] for idx in batch_idx]
wplabel_batch = [self.wplabels[idx] for idx in batch_idx]
# transform a list of sampled id sequences to a batch matrix of a fixed size by trimming and padding
# self.max_length-2: leave space for [SEP] and [CLS]
input_ids = np.zeros((size, self.max_length - 2), dtype=int)
for i, wpid in enumerate(wpid_batch):
l = min([len(wpid), self.max_length - 2])
input_ids[i, :l] = wpid[:l]
# add [SEP] at the end of each sentence (just before padding)
flipped_input_ids = self._flip_nonzero_columns(input_ids)
flipped_input_ids = np.c_[
np.tile(self.SEP, input_ids.shape[0]), flipped_input_ids]
input_ids = self._flip_nonzero_columns(flipped_input_ids)
# add [CLS] at the beginning of each sentence
input_ids = np.c_[np.tile(self.CLS, input_ids.shape[0]),
input_ids]
# all tokens belong to the same sentence => all zeros
token_type_ids = np.zeros((size, self.max_length), dtype=int)
# highlight padding (zeros)
attention_mask = (input_ids != 0).astype(int)
# transform a list of sampled labels to a batch matrix of a fixed size by trimming and padding
# input_ids[i, 1:l+1] : the 0th item corresponds to the [CLS] token - use 0 label for it
labels = np.zeros((size, self.max_length), dtype=int)
for i, wplabel in enumerate(wplabel_batch):
l = min([len(wplabel), self.max_length - 2])
labels[i, 1:l + 1] = wplabel[:l]
yield SrcComplexityDataset.Batch(
input_ids=input_ids,
token_type_ids=token_type_ids,
attention_mask=attention_mask,
labels=labels,
)
def get_dataloader(opt, distributed):
input_size = opt.input_size
crop_ratio = opt.crop_ratio if opt.crop_ratio > 0 else 0.875
resize = int(math.ceil(input_size / crop_ratio))
transform_test = transforms_cv.Compose([
transforms_cv.Resize((resize, resize)),
transforms_cv.CenterCrop(input_size),
transforms_cv.ToTensor(),
transforms_cv.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
val_dataset = ImageNet(opt.data_dir, train=False, transform=transform_test)
sampler = make_data_sampler(val_dataset, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler, batch_size=opt.batch_size, drop_last=False)
val_loader = data.DataLoader(val_dataset, batch_sampler=batch_sampler, num_workers=opt.num_workers)
return val_loader
def get_latents(self, dset):
self.enc.eval()
collected_latents = []
determin_dl = data.DataLoader(dset,
batch_sampler=data.BatchSampler(
data.SequentialSampler(dset),
self.batch_size,
drop_last=False),
pin_memory=False)
for idx, (xb, yb, tb) in enumerate(determin_dl):
batch_latents = self.enc(xb)
batch_latents = batch_latents.view(batch_latents.shape[0],
-1).detach().cpu().numpy()
collected_latents.append(batch_latents)
collected_latents = np.concatenate(collected_latents, axis=0)
return collected_latents
def train_network(self, X_inputs, Y_labels):
optimizer = optim.Adam(self.neural_network.parameters())
X_inputs = torch.from_numpy(X_inputs).double()
Y_labels = torch.from_numpy(Y_labels).double().view(len(Y_labels), 1)
self.neural_network.train(True)
for iteration in range(self.nb_iters):
for batch in tdata.BatchSampler(
tdata.RandomSampler(range(len(X_inputs)), replacement=False),
batch_size=self.batch_size, drop_last=False):
optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = self.neural_network(X_inputs[batch])
loss = nn.MSELoss(reduction="mean")(outputs, Y_labels[batch])
loss.backward()
optimizer.step()
def main(self, clevr_dir, preproc_dir, results_loc, log_loc=None):
logging.basicConfig(level=logging.INFO)
utils.cuda_message()
np.printoptions(linewidth=139)
clevr_fs = open_fs(clevr_dir, create=False)
preproc_fs = open_fs(preproc_dir, create=True)
dataset = datasets.TaskDataset(clevr_fs, preproc_fs, "train")
total_words = len(dataset.word_ix) + 1
logging.info("Total words: %s", total_words)
sampler = data.BatchSampler(data.RandomSampler(dataset), 32, False)
net = mac.MACNet(mac.MACRec(12, 512),
total_words).to(config.torch_device())
opt = torch.optim.Adam(net.parameters())
if log_loc:
now = datetime.datetime.now()
log_dir = f"{log_loc}/new-{now}"
writer = tensorboardX.SummaryWriter(log_dir)
else:
writer = None
step = 0
rolling_accuracy = 0
for epoch in range(10):
bar = tqdm(sampler)
for batch_ix in bar:
opt.zero_grad()
images, qns, qn_lens, answers = dataset[batch_ix]
predictions = net(images, qns, qn_lens)
loss = functional.cross_entropy(predictions, answers)
loss.backward()
opt.step()
hard_preds = np.argmax(predictions.detach().cpu().numpy(), 1)
accuracy = (
hard_preds == answers.detach().cpu().numpy()).mean()
if writer is not None:
writer.add_scalar("loss", loss.item(), step)
writer.add_scalar("accuracy", accuracy, step)
rolling_accuracy = rolling_accuracy * 0.95 + accuracy * 0.05
bar.set_description("Accuracy: {}".format(rolling_accuracy))
step += 1
def __init__(self,
dataset: data.Dataset,
mask: bool,
batch_size: int,
initial_temperature: float,
drop_last: bool = False,
device='cpu'):
super().__init__()
self._dataset = dataset
self.mask = mask
self.batch_size = batch_size
self.drop_last = drop_last
self.device = device
self._temperature = initial_temperature
# * TODO: A better than random sampler that take into account sample length
self._sampler = data.BatchSampler(data.RandomSampler(
self._dataset, replacement=False),
batch_size=self.batch_size,
drop_last=self.drop_last)
def main():
args = parse_args()
config = load_config(os.path.join(args.models_path, args.exp_name))
if config['model'] == 'TextTransposeModel':
test_set = DatasetTextImages(path=args.test_data_path, patch_size=None,
aug_resize_factor_range=None, scale=config['scale'])
else:
test_set = DatasetFromSingleImages(path=args.test_data_path, patch_size=None,
aug_resize_factor_range=None, scale=config['scale'])
batch_sampler = Data.BatchSampler(
sampler=Data.SequentialSampler(test_set),
batch_size=1,
drop_last=True
)
evaluation_data_loader = Data.DataLoader(dataset=test_set, num_workers=0, batch_sampler=batch_sampler)
trainer = Trainer(name=args.exp_name, models_root=args.models_path, resume=True)
trainer.load_best()
psnr = PSNR(name='PSNR', border=config['border'])
tic = time.time()
count = 0
for batch in tqdm(evaluation_data_loader):
output = trainer.predict(batch=batch)
psnr.update(batch[1], output)
count += 1
toc = time.time()
print('FPS: {}, SAMPLES: {}'.format(float(count) / (toc - tic), count))
print('PSNR: {}'.format(psnr.get()))
def __init__(self, args):
self.device = torch.device(args.device)
# network
net_name = '_'.join(('yolo3', args.network, args.dataset))
self.save_prefix = net_name
self.net = get_model(net_name, pretrained_base=True)
if args.distributed:
self.net = torch.nn.SyncBatchNorm.convert_sync_batchnorm(self.net)
if args.resume.strip():
logger.info("Resume from the model {}".format(args.resume))
self.net.load_state_dict(torch.load(args.resume.strip()))
else:
logger.info("Init from base net {}".format(args.network))
classes, anchors = self.net.num_class, self.net.anchors
self.net.set_nms(nms_thresh=0.45, nms_topk=400)
if args.label_smooth:
self.net._target_generator._label_smooth = True
self.net.to(self.device)
if args.distributed:
self.net = torch.nn.parallel.DistributedDataParallel(
self.net, device_ids=[args.local_rank], output_device=args.local_rank)
# dataset and dataloader
train_dataset = get_train_data(args.dataset, args.mixup)
width, height = args.data_shape, args.data_shape
batchify_fn = Tuple(*([Stack() for _ in range(6)] + [Pad(axis=0, pad_val=-1) for _ in range(1)]))
train_dataset = train_dataset.transform(
YOLO3DefaultTrainTransform(width, height, classes, anchors, mixup=args.mixup))
args.per_iter = len(train_dataset) // (args.num_gpus * args.batch_size)
args.max_iter = args.epochs * args.per_iter
if args.distributed:
sampler = data.DistributedSampler(train_dataset)
else:
sampler = data.RandomSampler(train_dataset)
train_sampler = data.sampler.BatchSampler(sampler=sampler, batch_size=args.batch_size,
drop_last=False)
train_sampler = IterationBasedBatchSampler(train_sampler, num_iterations=args.max_iter)
if args.no_random_shape:
self.train_loader = data.DataLoader(train_dataset, batch_sampler=train_sampler, pin_memory=True,
collate_fn=batchify_fn, num_workers=args.num_workers)
else:
transform_fns = [YOLO3DefaultTrainTransform(x * 32, x * 32, classes, anchors, mixup=args.mixup)
for x in range(10, 20)]
self.train_loader = RandomTransformDataLoader(transform_fns, train_dataset, batch_sampler=train_sampler,
collate_fn=batchify_fn, num_workers=args.num_workers)
if args.eval_epoch > 0:
# TODO: rewrite it
val_dataset, self.metric = get_test_data(args.dataset)
val_batchify_fn = Tuple(Stack(), Pad(pad_val=-1))
val_dataset = val_dataset.transform(YOLO3DefaultValTransform(width, height))
val_sampler = make_data_sampler(val_dataset, False, args.distributed)
val_batch_sampler = data.BatchSampler(val_sampler, args.test_batch_size, False)
self.val_loader = data.DataLoader(val_dataset, batch_sampler=val_batch_sampler,
collate_fn=val_batchify_fn, num_workers=args.num_workers)
# optimizer and lr scheduling
self.optimizer = optim.SGD(self.net.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=args.wd)
if args.lr_mode == 'cos':
self.scheduler = WarmupCosineLR(optimizer=self.optimizer, T_max=args.max_iter,
warmup_factor=args.warmup_factor, warmup_iters=args.warmup_iters)
elif args.lr_mode == 'step':
lr_decay = float(args.lr_decay)
milestones = sorted([float(ls) * args.per_iter for ls in args.lr_decay_epoch.split(',') if ls.strip()])
self.scheduler = WarmupMultiStepLR(optimizer=self.optimizer, milestones=milestones, gamma=lr_decay,
warmup_factor=args.warmup_factor, warmup_iters=args.warmup_iters)
else:
raise ValueError('illegal scheduler type')
self.args = args
input_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([.485, .456, .406], [.229, .224, .225]),
])
data_kwargs = {
'base_size': args.base_size,
'crop_size': args.crop_size,
'transform': input_transform
}
val_dataset = get_segmentation_dataset(args.dataset,
split=args.split,
mode=args.mode,
**data_kwargs)
sampler = make_data_sampler(val_dataset, False, distributed)
batch_sampler = data.BatchSampler(sampler=sampler,
batch_size=args.batch_size,
drop_last=False)
val_data = data.DataLoader(val_dataset,
shuffle=False,
batch_sampler=batch_sampler,
num_workers=args.num_workers)
metric = SegmentationMetric(val_dataset.num_class)
metric = validate(model, val_data, metric, device)
ptutil.synchronize()
pixAcc, mIoU = ptutil.accumulate_metric(metric)
if ptutil.is_main_process():
print('pixAcc: %.4f, mIoU: %.4f' % (pixAcc, mIoU))
def pseudo_label_train(self, pseudo_label_dset, mask, num_epochs):
self.enc.train()
pseudo_label_dl = data.DataLoader(
pseudo_label_dset,
batch_sampler=data.BatchSampler(
data.RandomSampler(pseudo_label_dset),
self.batch_size,
drop_last=False),
pin_memory=False)
for epoch in range(num_epochs):
epoch_loss = 0
best_loss = np.inf
pred_list = []
idx_list = []
for batch_idx, (xb, yb, idx) in enumerate(pseudo_label_dl):
batch_mask = mask[idx]
latent = self.enc(xb)
latent = noiseify(latent, ARGS.noise)
if batch_mask.any():
try:
pseudo_label_pred = self.mlp(latent[:, :, 0, 0])
except ValueError:
set_trace()
pseudo_label_loss = self.pseudo_label_lf(
pseudo_label_pred[batch_mask],
yb.long()[batch_mask])
else:
pseudo_label_loss = torch.tensor(0, device=self.device)
if not batch_mask.all():
latents_to_rec_train = latent[~batch_mask]
rec_pred = self.dec(latents_to_rec_train)
rec_loss = self.rec_lf(
rec_pred, xb[~batch_mask]) / (~batch_mask).sum()
else:
rec_loss = torch.tensor(0, device=pseudo_label_dset.device)
loss = pseudo_label_loss + rec_loss
if math.isnan(loss): set_trace()
loss.backward()
self.enc_opt.step()
self.enc_opt.zero_grad()
self.dec_opt.step()
self.dec_opt.zero_grad()
self.mlp_opt.step()
self.mlp_opt.zero_grad()
pred_list.append(
pseudo_label_pred.argmax(axis=1).detach().cpu().numpy())
idx_list.append(idx.detach().cpu().numpy())
if ARGS.test: break
if ARGS.test:
pred_array_ordered = dummy_labels(self.num_classes,
len(pseudo_label_dset))
break
pred_array = np.concatenate(pred_list)
idx_array = np.concatenate(idx_list)
pred_array_ordered = np.array([
item[0] for item in sorted(zip(pred_array, idx_array),
key=lambda x: x[1])
])
if epoch_loss < best_loss:
best_loss = epoch_loss
count = 0
else:
count += 1
if count > 4: break
return pred_array_ordered
print('System start to load data...')
t0 = time()
train_data, val_data, test_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_lm_id_x.py' % (conf.out_path, conf.data_name))
log = Logging('%s/train_%s_att2seq_id_X.py' %
(conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_att2seq_id_X' % (conf.out_path,
conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(\
range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False)
review_val_dataset = data_utils.TestData(val_data)
review_val_sampler = data.BatchSampler(data.RandomSampler(\
range(review_val_dataset.length)), batch_size=conf.batch_size, drop_last=False)
review_test_dataset = data_utils.TestData(test_data)
review_test_sampler = data.BatchSampler(data.RandomSampler(\
range(review_test_dataset.length)), batch_size=conf.batch_size, drop_last=False)
# Start Training !!!
max_bleu = 0.0
for epoch in range(1, conf.train_epochs + 1):
t0 = time()
model.train()
))
model.cuda()
optimizer = torch.optim.Adam(model.parameters(), lr=conf.learning_rate)
import pdb
pdb.set_trace()
############################## PREPARE DATASET ##############################
print('System start to load data...')
t0 = time()
train_data, val_data, test_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
test_dataset = data_utils.TestData(test_data)
test_batch_sampler = data.BatchSampler(data.RandomSampler(
range(test_dataset.length)),
batch_size=1,
drop_last=False)
print('test dataset length:%d' % test_dataset.length)
word_dict = constructDict()
t0 = time()
count = 0
bleu_score = []
bleu_list_1, bleu_list_2, bleu_list_3, bleu_list_4 = [], [], [], []
rouge_1_list, rouge_2_list, rouge_L_list = [], [], []
out_probit = []
target = []
for batch_idx_list in test_batch_sampler:
user_list, item_list, review_input_list, _, real_review_list = test_dataset.get_batch(
batch_idx_list)
{},
[],
{"data_source": dummy_dataset},
data.Sampler(data_source=dummy_dataset),
id="SamplerConf",
),
pytest.param(
"utils.data.sampler",
"BatchSampler",
{
"batch_size": 4,
"drop_last": False
},
[],
{"sampler": dummy_sampler},
data.BatchSampler(
sampler=dummy_sampler, batch_size=4, drop_last=False),
id="BatchSamplerConf",
),
pytest.param(
"utils.data.sampler",
"RandomSampler",
{},
[],
{"data_source": dummy_dataset},
data.RandomSampler(data_source=dummy_dataset),
id="RandomSamplerConf",
),
pytest.param(
"utils.data.sampler",
"SequentialSampler",
{},
test_data = KBDataset(os.path.join("data", args.task_dir, 'test2id.txt'),
args.cuda)
chk = DataChecker(args.task_dir)
# init data loader
train_loader = data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True)
valid_loader = data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=True)
test_loader = data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=True)
test_sampler = data.BatchSampler(data.RandomSampler(test_data),
batch_size=args.sample_size,
drop_last=True)
ent_size = chk.ent_size
rel_size = chk.rel_size
from models.seq2seq_egreedy import Seq2Seq
gen = Seq2Seq(args, rel_size, ent_size)
g_opt = None
d_opt = None
if args.optim == "SGD":
g_opt = optim.SGD(gen.parameters(), lr=args.lr, momentum=0.9)
elif args.optim == "Adam":
g_opt = optim.Adam(gen.parameters())
def get_sampler(self, batch_size, drop_last=False, **kwargs): return data.BatchSampler(BucketSampler(self, batch_size, len_step=3), batch_size, drop_last=drop_last)
############################## LOAD MODEL ##############################
from language_model import language_model
model = language_model()
model.load_state_dict(
torch.load(
'/content/drive/My Drive/task/aspect_based_rs/out/amazon_clothing/train_amazon_clothing_language_model_id_0X.mod'
))
model.cuda()
########################### TEST STAGE #####################################
test_dataset = data_utils.TrainData(test_data)
test_batch_sampler = data.BatchSampler(data.SequentialSampler(
range(test_dataset.length)),
batch_size=conf.batch_size,
drop_last=False)
# Start Training !!!
t0 = time()
model.eval()
beam_size = 4
for batch_idx_list in test_batch_sampler:
user, item, _, review_input, review_output = test_dataset.get_batch(
batch_idx_list)
generate(batch_idx_list, user, item,
tensorToScalar(review_output.transpose(0, 1)).tolist())
def main():
opt = parser.parse_args()
print(opt)
print(torch.__version__)
opt.seed = 1337
random.seed(opt.seed)
torch.manual_seed(opt.seed)
np.random.seed(opt.seed)
torch.cuda.manual_seed(opt.seed)
series = [
f for f in os.listdir(opt.train_path[0])
if os.path.isdir(os.path.join(opt.train_path[0], f))
]
series.sort()
print("===> Building model")
enc_layers = [1, 2, 2, 4]
dec_layers = [1, 1, 1, 1]
number_of_channels = [int(8 * 2**i) for i in range(1, 1 + len(enc_layers))]
model = UNet(depth=len(enc_layers),
encoder_layers=enc_layers,
decoder_layers=dec_layers,
number_of_channels=number_of_channels,
number_of_outputs=3)
model.apply(weight_init.weight_init)
model = torch.nn.DataParallel(module=model, device_ids=range(opt.gpus))
trainer = train.Trainer(model=model,
name=opt.name,
models_root=opt.models_path,
rewrite=False)
trainer.cuda()
gc.collect()
opt.seed = 1337 # random.randint(1, 10000)
random.seed(opt.seed)
torch.manual_seed(opt.seed)
np.random.seed(opt.seed)
torch.cuda.manual_seed(opt.seed)
cudnn.benchmark = True
print("===> Loading datasets")
print('Train data:', opt.train_path)
series_val = [
'BraTS19_2013_0_1', 'BraTS19_2013_12_1', 'BraTS19_2013_16_1',
'BraTS19_2013_2_1', 'BraTS19_2013_23_1', 'BraTS19_2013_26_1',
'BraTS19_2013_29_1', 'BraTS19_CBICA_AAB_1', 'BraTS19_CBICA_AAP_1',
'BraTS19_CBICA_AMH_1', 'BraTS19_CBICA_AQD_1', 'BraTS19_CBICA_ATX_1',
'BraTS19_CBICA_AZH_1', 'BraTS19_CBICA_BHB_1', 'BraTS19_TCIA12_101_1',
'BraTS19_TCIA01_150_1', 'BraTS19_TCIA10_152_1', 'BraTS19_TCIA04_192_1',
'BraTS19_TCIA08_205_1', 'BraTS19_TCIA06_211_1', 'BraTS19_TCIA02_222_1',
'BraTS19_TCIA12_298_1', 'BraTS19_TCIA13_623_1', 'BraTS19_CBICA_ANV_1',
'BraTS19_CBICA_BBG_1', 'BraTS19_TMC_15477_1'
]
series_train = [f for f in series if f not in series_val]
print('Train {}'.format(series_train))
print('Val {}'.format(series_val))
train_set = dataloader.SimpleReader(
paths=opt.train_path,
patch_size=(144, 144, 128),
series=[
series_train,
] + [None for i in range(len(opt.train_path) - 1)],
annotation_path=opt.annotation_path,
images_in_epoch=8000,
patches_from_single_image=1)
val_set = dataloader.FullReader(path=opt.train_path[0], series=series_val)
training_data_loader = DataLoader(dataset=train_set,
num_workers=opt.threads,
batch_size=opt.batchSize,
shuffle=True,
drop_last=True,
worker_init_fn=worker_init_fn)
batch_sampler = Data.BatchSampler(sampler=Data.SequentialSampler(val_set),
batch_size=1,
drop_last=True)
evaluation_data_loader = DataLoader(dataset=val_set,
num_workers=0,
batch_sampler=batch_sampler)
criterion = [
loss.Dice_loss_joint(index=0, priority=1).cuda(),
loss.BCE_Loss(index=0, bg_weight=1e-2).cuda(),
]
print("===> Building model")
print("===> Training")
trainer.train(criterion=criterion,
optimizer=optim.Adam,
optimizer_params={"lr": 2e-5,
"weight_decay": 1e-6,
"amsgrad": True,
},
scheduler=torch.optim.lr_scheduler.StepLR,
scheduler_params={"step_size": 16000,
"gamma": 0.5,
},
training_data_loader=training_data_loader,
evaluation_data_loader=evaluation_data_loader,
split_into_tiles=False,
pretrained_weights=None,
train_metrics=[metrics.Dice(name='Dice', input_index=0, target_index=0, classes=4), \
],
val_metrics=[metrics.Dice(name='Dice', input_index=0, target_index=0, classes=4),
metrics.Hausdorff_ITK(name='Hausdorff_ITK', input_index=0, target_index=0, classes=4),
],
track_metric='Dice',
epoches=opt.nEpochs,
default_val=np.array([0, 0, 0, 0, 0]),
comparator=lambda x, y: np.min(x) + np.mean(x) > np.min(y) + np.mean(y),
eval_cpu=False,
continue_form_pretraining=False
)
args = parse_args()
print("start to load data")
user_num, item_num, train_data, val_negative_dict, val_negative, train_mat, val_mat = data_prepare.load_all(
args)
print('data loaded')
hrs, ngs, losss = [], [], []
pmfloss = PMFLoss(user_num, item_num).cuda()
Train_data = data_prepare.Test_data(train_data, item_num, train_mat, 5)
Train_data.ng_test_sample()
hrmax = 0.0
ndcgmax = 0.0
for epoch in range(1, 100):
sampler = data.BatchSampler(data.RandomSampler(range(Train_data.idx)),
batch_size=10240,
drop_last=False)
pmfloss.train()
loss_sum = 0.0
for u_list in sampler:
user_list, item_list, label_list = Train_data.get_batch(u_list)
loss = pmfloss.forward(user_list, item_list, label_list)
pmfloss.optim.zero_grad()
loss.backward()
pmfloss.optim.step()
loss_sum += loss.item()
pmfloss.eval()
hrtmp, ndcgtmp = metrics_pmf(pmfloss, val_negative_dict, val_negative,
10)
str_log = 'epoch:{}----loss:{}----hr:{}----ndcg{}'.format(
epoch, loss_sum, hrtmp, ndcgtmp)
print('System start to load data...')
t0 = time()
train_data, val_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### TRAINING STAGE ##################################
check_dir('%s/train_log' % conf.out_path)
log = Logging('%s/train_%s_nrms.log' % (conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_nrms.mod' % (conf.out_path, conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
val_dataset = data_utils.TestData(val_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(
range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False)
val_batch_sampler = data.BatchSampler(data.SequentialSampler(
range(val_dataset.length)), batch_size=conf.batch_size, drop_last=True)
# Start Training !!!
max_auc = 0
for epoch in range(1, conf.train_epochs+1):
t0 = time()
model.train()
train_loss = []
count = 0
for batch_idx_list in train_batch_sampler:
his_input_title, pred_input_title, labels = \
train_dataset._get_batch(batch_idx_list)
'transform': input_transform
}
else:
data_kwargs = {
'base_size': config.DATASET.BASE_SIZE,
'crop_size': config.DATASET.CROP_SIZE,
'transform': None
}
val_dataset = get_segmentation_dataset(config.DATASET.NAME,
split=config.TEST.TEST_SPLIT,
mode=config.TEST.MODE,
**data_kwargs)
sampler = make_data_sampler(val_dataset, False, is_distributed)
batch_sampler = data.BatchSampler(sampler=sampler,
batch_size=config.TEST.TEST_BATCH_SIZE,
drop_last=False)
val_data = data.DataLoader(val_dataset,
shuffle=False,
batch_sampler=batch_sampler,
num_workers=config.DATASET.WORKERS)
metric = SegmentationMetric(val_dataset.NUM_CLASS)
model = get_segmentation_model(config.TEST.MODEL_NAME,
nclass=val_dataset.NUM_CLASS)
if not os.path.exists(config.TEST.PRETRAINED):
raise RuntimeError('cannot found the pretrained file in {}'.format(
config.TEST.PRETRAINED))
model.load_state_dict(torch.load(config.TEST.PRETRAINED))
model.keep_shape = True if config.TEST.MODE == 'testval' else False
model.to(device)
