def train(LOOP):
# setup model
model.train()
model.to(DEVICE)
# load model ckpt from the previous trained LOOP
if LOOP:
ckpt = os.path.join(EXPT_DIR, 'ckpt_{}.pth'.format(LOOP - 1))
model.load_state_dict(torch.load(ckpt))
# load selected indices
labeled = os.path.join(EXPT_DIR, 'labeled.txt')
f = open(labeled, 'r')
indices = f.readlines()
indices = [int(x.strip()) for x in indices]
f.close()
# setup dataloader
dataloader = DataLoader(cifar_train,
batch_size=64,
sampler=SubsetRandomSampler(indices))
loss_fn = nn.CrossEntropyLoss()
opt = optim.Adam(model.parameters(),
lr=params['learning_rate'],
weight_decay=params['weight_decay'])
for epoch in range(epochs):
for x, y in dataloader:
x, y = x.to(DEVICE), y.to(DEVICE)
y_ = model(x)
loss = loss_fn(y_, y)
opt.zero_grad()
loss.backward()
opt.step()
# save ckpt
ckpt = os.path.join(EXPT_DIR, 'ckpt_{}.pth'.format(LOOP))
torch.save(model.state_dict(), ckpt)
return
def get_sampler_classifier(dataloader, seed, proportion_training):
if proportion_training == 1:
return None
labels = np.array(dataloader.dataset.train_labels)
unique_labels = np.unique(labels)
n_labels = len(unique_labels)
num_training_samples_per_label = int(proportion_training * len(labels) /
n_labels)
training_subset_idx = []
for label in unique_labels:
label_set = np.where(labels == label)[0]
label_set_len = len(label_set)
# Subset indices for training
np.random.seed(seed)
label_training_subset = np.random.choice(
label_set, size=num_training_samples_per_label, replace=False)
training_subset_idx += list(label_training_subset)
return SubsetRandomSampler(training_subset_idx)
def make_data_loader(*tensors, N, batch_size,
num_batches) -> torch.utils.data.DataLoader:
"""
Creates DataLoader, which samples num_batches batches of sequences with shape (batch_size, N) from tensors
:param tensors: tensors to sample batches from
:param N: length of sampled subsequences
:param batch_size: batch size
:param num_batches: amount of batches to sample
:return: DataLoader
"""
assert tensors[0].shape[
0] % N == 0, "length of tensors has to be dividable by N!"
M = tensors[0].shape[0] // N
dataset = SequenceDataset(*tensors, N=N)
indices = np.random.choice(N * (M - 1),
size=batch_size * num_batches,
replace=False)
base_sampler = SubsetRandomSampler(indices)
loader = DataLoader(dataset, sampler=base_sampler, batch_size=batch_size)
return loader
def create_dataloader(args, dataset, batch_size, eval_slice_dataset=None, world_size=None, rank=None):
logger = logging_utils.get_logger(args)
if eval_slice_dataset is not None and not args.run_config.distributed:
indices = get_eval_slice_subset_indices(args, eval_slice_dataset=eval_slice_dataset, dataset=dataset)
# Form sampler with for indices from eval_slice_dataset
sampler = SubsetRandomSampler(indices)
elif args.run_config.distributed:
# wrap dataset object to use a subsetsampler with distributed
if eval_slice_dataset is not None:
indices = get_eval_slice_subset_indices(args, eval_slice_dataset=eval_slice_dataset, dataset=dataset)
dataset = DistributedIndicesWrapper(dataset, torch.tensor(indices))
sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank)
else:
sampler = None
dataloader = DataLoader(dataset, batch_size=batch_size,
shuffle=(sampler is None),
sampler=sampler,
num_workers=args.run_config.dataloader_threads,
pin_memory=False)
return dataloader, sampler
def get_restrictedImageNetValidationTestSplit(val_split=True, batch_size=128, shuffle=False, augm_type='test', num_workers=8, size=224):
idxs = np.loadtxt('idxs_rimgnet.txt', dtype=int)
if not val_split:
all_idcs = np.arange(10150)
idxs = np.setdiff1d(all_idcs, idxs) #test idcs
if shuffle:
sampler = SubsetRandomSampler(idxs)
else:
sampler = SubsetSampler(idxs)
transform = get_imageNet_augmentation(type=augm_type, out_size=size)
path = get_imagenet_path()
dataset = RestrictedImageNet(path=path, split='val',
transform=transform, balanced=False)
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=sampler,
num_workers=num_workers)
return loader
def __init__(self, concat_dataset, batch_size, drop_last, patial_batches_ok):
super(MTBatchSampler, self).__init__(concat_dataset)
self.task_idx_bins = []
offset = 0
for ds in concat_dataset.datasets:
self.task_idx_bins.append(np.arange(offset, offset + len(ds)))
offset += len(ds)
if not isinstance(batch_size, int) or isinstance(batch_size, bool) or \
batch_size <= 0:
raise ValueError("batch_size should be a positive integer value, "
"but got batch_size={}".format(batch_size))
if not isinstance(drop_last, bool):
raise ValueError("drop_last should be a boolean value, but got "
"drop_last={}".format(drop_last))
self.samplers = [SubsetRandomSampler(idx) for idx in self.task_idx_bins]
self.batch_size = batch_size
self.drop_last = drop_last
self.partial_batches_ok = patial_batches_ok
def fit(self, zeta, xu, nb_iter=100, batch_size=None, lr=1e-3):
if self.prior and 'l2_penalty' in self.prior:
self.optim = Adam(self.parameters(),
lr=lr,
weight_decay=self.prior['l2_penalty'])
else:
self.optim = Adam(self.parameters(), lr=lr)
set_size = xu.shape[0]
batch_size = set_size if batch_size is None else batch_size
batches = list(
BatchSampler(SubsetRandomSampler(range(set_size)), batch_size,
True))
for n in range(nb_iter):
for batch in batches:
self.optim.zero_grad()
loss = -self.elbo(zeta[batch], xu[batch], batch_size, set_size)
loss.backward()
self.optim.step()
def get_dataloader(is_train=True,
indeces=None,
batch_size=32,
num_workers=0,
data_path=DATA_FOLDER):
# https://discuss.pytorch.org/t/train-on-a-fraction-of-the-data-set/16743/6
dataset = get_dataset(is_train, data_path)
if indeces is None:
data_loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=is_train,
num_workers=num_workers)
else:
data_loader = DataLoader(dataset,
batch_size=batch_size,
shuffle=is_train,
num_workers=num_workers,
sampler=SubsetRandomSampler(indeces))
return data_loader
def _update_offpolicy(self):
if len(self.memory) > self.batch_size:
for _ in range(self.epochs):
indices = torch.randperm(len(self.memory))[:self.batch_size]
indices = len(self.memory) - indices - 1
dataloader = DataLoader(self.memory,
self.batch_size,
sampler=SubsetRandomSampler(indices),
num_workers=8)
if self.q_update == 2:
self.q_update = 1
for states, actions, rewards, dones, next_states in dataloader:
self._training_q(states.to(self.device),
actions.to(self.device),
rewards.to(self.device),
dones.to(self.device),
next_states.to(self.device))
self._training_policy(states.to(self.device))
self.target_soft_q1 = copy_parameters(
self.soft_q1, self.target_soft_q1, self.soft_tau)
self.target_soft_q2 = copy_parameters(
self.soft_q2, self.target_soft_q2, self.soft_tau)
self.target_policy = copy_parameters(
self.policy, self.target_policy, self.soft_tau)
else:
self.q_update = 2
for states, actions, rewards, dones, next_states in dataloader:
self._training_q(states.to(self.device),
actions.to(self.device),
rewards.to(self.device),
dones.to(self.device),
next_states.to(self.device))
self.target_soft_q1 = copy_parameters(
self.soft_q1, self.target_soft_q1, self.soft_tau)
self.target_soft_q2 = copy_parameters(
self.soft_q2, self.target_soft_q2, self.soft_tau)
def train():
# setup model
model.train()
model.to(DEVICE)
if RESUME_FROM:
resume_from = os.path.join(EXPT_DIR, RESUME_FROM)
model.load_state_dict(torch.load(resume_from))
# load selected indices
labeled = load(os.path.join(EXPT_DIR, 'labeled.pkl'))
indices = list(labeled.keys())
# change dataset object's target attribute to labels
# this is used for pollution study
for ix in labeled:
mnist_train.targets[ix] = labeled[ix]
# setup dataloader
dataloader = DataLoader(mnist_train,
batch_size=64,
sampler=SubsetRandomSampler(indices))
loss_fn = nn.CrossEntropyLoss()
opt = optim.Adam(model.parameters(),
lr=params['learning_rate'],
weight_decay=params['weight_decay'])
for epoch in range(epochs):
for x, y in dataloader:
x, y = x.to(DEVICE), y.to(DEVICE)
y_ = model(x)
loss = loss_fn(y_, y)
opt.zero_grad()
loss.backward()
opt.step()
# save ckpt
ckpt = os.path.join(EXPT_DIR, CKPT_FILE)
torch.save(model.state_dict(), ckpt)
return
def get_dataloader(synthetic_dataset, real_dataset,
height, width, batch_size,
workers, is_train, keep_ratio):
num_synthetic_dataset = len(synthetic_dataset)
num_real_dataset = len(real_dataset)
synthetic_indices = list(np.random.permutation(num_synthetic_dataset))
synthetic_indices = synthetic_indices[num_real_dataset:]
real_indices = list(np.random.permutation(
num_real_dataset) + num_synthetic_dataset)
concated_indices = synthetic_indices + real_indices
assert len(concated_indices) == num_synthetic_dataset
sampler = SubsetRandomSampler(concated_indices)
concated_dataset = ConcatDataset([synthetic_dataset, real_dataset])
print('total image: ', len(concated_dataset))
data_loader = DataLoader(concated_dataset, batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True, drop_last=True, sampler=sampler,
collate_fn=AlignCollate(imgH=height, imgW=width, keep_ratio=keep_ratio))
return concated_dataset, data_loader
def main():
trainset = torchvision.datasets.FashionMNIST('../data', train=True, download=True,
transform=transforms.Compose([
lambda x: transforms.functional.to_grayscale(x, num_output_channels=3),
transforms.Resize(32),
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
]))
dataset_indices = list(range(len(trainset)))
dataset_indices = dataset_indices[2000:3000]
sampler = SubsetRandomSampler(dataset_indices)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=defs.batch_size,
drop_last=False, num_workers=4, pin_memory=True, sampler=sampler)
exit(0)
if opt.data == 'cifar100':
downloaded_list = [
['train', '16019d7e3df5f24257cddd939b257f8d'],
]
root = os.path.join(os.getenv("HOME"), 'data')
base_folder = 'cifar-100-python'
# now load the picked numpy arrays
data = list()
targets = list()
for file_name, checksum in downloaded_list:
file_path = os.path.join(root, base_folder, file_name)
with open(file_path, 'rb') as f:
if sys.version_info[0] == 2:
entry = pickle.load(f)
else:
entry = pickle.load(f, encoding='latin1')
data.append(entry['data'])
if 'labels' in entry:
targets.extend(entry['labels'])
else:
targets.extend(entry['fine_labels'])
def fit(self,
target,
input,
nb_epochs=1000,
batch_size=32,
lr=1e-3,
l2=1e-32,
verbose=True,
preprocess=True):
if preprocess:
self.init_preprocess(target, input)
target = transform(target, self.target_trans)
input = transform(input, self.input_trans)
self.optim = Adam(self.parameters(), lr=lr, weight_decay=l2)
set_size = input.shape[0]
batch_size = set_size if batch_size is None else batch_size
for n in range(nb_epochs):
batches = list(
BatchSampler(SubsetRandomSampler(range(set_size)), batch_size,
True))
for batch in batches:
self.optim.zero_grad()
_output = self.forward(input[batch])
loss = self.criterion(_output, target[batch])
loss.backward()
self.optim.step()
if verbose:
if n % 50 == 0:
output = self.forward(input)
print('Epoch: {}/{}.............'.format(n, nb_epochs),
end=' ')
print("Loss: {:.4f}".format(self.criterion(output,
target)))
def data_loaders_split(params: dict):
"""
Creats data loaders for train, validation and test sections of data
Args:
params: dictionary of params to input to data loader:
dataset, batch_size, num_workers, collate_fn
Returns:
data_loaders: dictionary of data loaders for train, val, test splits of data
"""
# Create a dictionary to collect
index_names = get_split_index_names()
data_loaders = dict.fromkeys(index_names)
for key in data_loaders:
# Make sampler and data loader
cur_prop_name = index_names[key]
cur_split_indices = getattr(params['dataset'], cur_prop_name)
cur_sampler = SubsetRandomSampler(cur_split_indices)
data_loaders[key] = torch.utils.data.DataLoader(sampler=cur_sampler,
**params)
return data_loaders
def __init__(
self,
root_dir,
dataset=None,
k_shot=None,
transform=None,
batch_size=64,
online_crop=False,
):
self.traindataset = CustomDataset(
root_dir,
dataset_selection=dataset,
k_shot=k_shot,
split=True,
train_valid="train",
transform=transform,
)
self.validdataset = CustomDataset(
root_dir,
dataset_selection=dataset,
k_shot=k_shot,
split=True,
train_valid="valid",
transform=transform,
)
self.batch_size = batch_size
self.k_shot = k_shot
self.images_indices = self.traindataset.selectKshots()
self.meta_sampler = SubsetRandomSampler(self.images_indices)
self.train_loader = DataLoader(self.traindataset,
batch_size=self.batch_size,
sampler=self.meta_sampler)
self.val_loader = DataLoader(self.validdataset,
batch_size=self.batch_size)
def get_mnist_subset_loader(train: bool, path: str, c1: int,
c2: int) -> Tuple[DataLoader, int]:
"""Return an MNIST dataloader for the two specified classes.
Args:
train (bool): Should this be a training set or validation set
path (str): The directory in which to store/find the MNIST dataset
c1 (int): a number in [0, 9] denoting a MNIST class/number
c2 (int): a number in [0, 9] denoting a MNIST class/number
Returns:
Tuple[DataLoader, int]: Return a dataloader and its size
"""
# All inputs must be converted into torch tensors, and the normalization values
# have been precomputed and provided below.
mnist_transforms = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, )),
])
dataset = MNIST(root=path,
train=train,
download=True,
transform=mnist_transforms)
# Grab indices for the two classes we care about
idx_class1 = [i for i, t in enumerate(dataset.targets) if t == c1]
idx_class2 = [i for i, t in enumerate(dataset.targets) if t == c2]
idxs = idx_class1 + idx_class2
size = len(idxs)
loader = DataLoader(dataset,
sampler=SubsetRandomSampler(idxs),
batch_size=size)
return loader, size
def get_dataloader(csv_files,
csv_labels,
name,
batch_size,
shuffle,
split=None,
directory="",
num_workers=4):
data_transforms = {
"train":
transforms.Compose((
transforms.Resize((256, 256)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
)),
"valid":
transforms.Compose((
transforms.Resize((256, 256)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
)),
}
sampler = None
dataset = MURA(directory, csv_files, csv_labels, data_transforms[name])
if split is not None:
dataset_size = len(dataset)
indices = list(range(dataset_size))
new_dataset_size = int(np.floor(split * dataset_size))
np.random.shuffle(indices)
splited = indices[:new_dataset_size]
sampler = SubsetRandomSampler(splited)
return DataLoader(dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=shuffle,
sampler=sampler)
def get_mnist_dataloaders(batch_size=128,
character_classes=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]):
"""
MNIST dataloader with (32, 32) sized images.
"""
# Resize images so they are a power of 2
all_transforms = transforms.Compose(
[transforms.Resize(32), transforms.ToTensor()])
# Get train and test data
train_data = datasets.MNIST('./data',
train=True,
download=True,
transform=all_transforms)
test_data = datasets.MNIST('./data', train=False, transform=all_transforms)
# Use SubsetRandomSampler, given a set of indices, to be the dataloader.
character_indices = {}
for i in range(len(train_data)):
label = train_data[i][1]
if label in character_indices:
character_indices[label].append(i)
else:
character_indices[label] = [i]
train_loaders = []
for value in character_classes:
data_loader = DataLoader(train_data,
batch_size=batch_size,
sampler=SubsetRandomSampler(
character_indices[value]))
train_loaders.append(data_loader)
test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True)
all_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
return train_loaders, test_loader, all_loader
def dataset_loader(data_dir, batch_size, test_batch_size, train_num=100, test_num=100):
path = os.path.join(data_dir, "mnist_correct/label_correct_index.npy")
label_correct_indices = list(np.load(path))
random.seed(1234)
random.shuffle(label_correct_indices)
train_indices = label_correct_indices[:train_num]
test_indices = label_correct_indices[5000:5000 + test_num]
train_loader = torch.utils.data.DataLoader(
datasets.MNIST(data_dir, train=False, download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])),
batch_size=batch_size, shuffle=False, sampler=SubsetRandomSampler(train_indices), drop_last=True)
test_loader = torch.utils.data.DataLoader(
Subset(datasets.MNIST(data_dir, train=False, transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])), test_indices),
batch_size=test_batch_size, shuffle=False)
return train_loader, test_loader
def train():
forecast_length = 3
backcast_length = 24
batch_size = 1 # greater than 4 for viz
f_b_dim = (forecast_length, backcast_length)
num_samples = 0
lr = 1e-3
# If the user specified a checkpoint, load the data
# and model from that checkpoint/run
if CHECKPOINT_NAME == "":
data_gen = data_generator(batch_size, backcast_length, forecast_length,
signal_type='seasonality', random=True)
ts = []
# user did not specify a previous checkpoint, generate new data
for grad_step, x, in enumerate(data_gen):
ts.append(x)
num_samples += len(x)
if num_samples >= 100000:
break
ts = np.concatenate(ts)
save_ts(ts)
else:
ts = np.load("data/" + RUN_NAME + "/dataset/timeseries.npy")
data = DatasetTS(ts, forecast_length, backcast_length)
net = NBeats(stacks=[TrendBlock, SeasonalityBlock, GenericNBeatsBlock],
f_b_dim=f_b_dim,
num_blocks_per_stack=[2,4,4],
thetas_dims=[[2,2], [8,8], [2,8]],
hidden_layer_dim=64)
optimiser = optim.Adam(net.parameters(), lr=lr)
print('--- Training ---')
epochs = 50
initial_epoch_step = load_model(net, optimiser)
ds_len = len(data)
train_length = int(np.ceil(ds_len*0.95))
train_sampler = SubsetRandomSampler(list(range(train_length)))
validation_sampler = SubsetRandomSampler(list(range(train_length + 1, ds_len)))
train_loader = torch.utils.data.DataLoader(data,
batch_size=100,
sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(data,
batch_size=4,
sampler=validation_sampler)
backcast_validation_data, target_validation = iter(validation_loader).next()
writer = SummaryWriter("data/" + RUN_NAME) if TENSORBOARD_ENABLE else None
for epoch in range(initial_epoch_step, epochs):
for grad_step, (x, target) in enumerate(train_loader):
optimiser.zero_grad()
net.train()
forecast, backcast = net(x)
loss = F.smooth_l1_loss(forecast, target)
temp = loss
# loss = loss * 100
#loss = F.smooth_l1_loss(forecast, target)
loss.backward()
optimiser.step()
if writer:
writer.add_scalar("loss/training_loss", temp, epoch)
with torch.no_grad():
# save_model(net, optimiser, epoch)
if not DISABLE_PLOT:
plot(net,
backcast_validation_data,
target_validation,
validation_loader,
backcast_length,
forecast_length,
epoch,
writer)
plt.close()
print("\nTarget : %s" % targets[i])
print("\nPredicted : %s" % dec[i])
print(
"=====================================================================\n"
)
# -------------------------- Eval on Test -------------------------- #
print('\n\n==================== Eval on Test ====================\n\n')
dataset = MyDataset(tokenizer,
args.data_dir,
'test',
max_len=args.max_seq_length)
# loader = DataLoader(dataset, batch_size=26, shuffle=True)
from torch.utils.data import SubsetRandomSampler
splr = SubsetRandomSampler([i for i in range(len(dataset))])
loader = DataLoader(
dataset,
batch_size=args.eval_batch_size, # 16,
sampler=splr,
shuffle=False, # sampler is mutually exclusive with shuffle
drop_last=True)
metric = datasets.load_metric('seqeval')
for i in range(len(loader)):
it = iter(loader)
batch = next(it)
outs = model.generate(input_ids=batch['source_ids'].cuda(),
attention_mask=batch['source_mask'].cuda(),
max_length=args.max_seq_length)
def train(self, batch_size=6):
# PPO algorithm
# Unroll rewards
rewards = np.array(self.rewards)
reward = 0
for i in reversed(range(len(self.rewards))):
rewards[i] += self.gamma * reward
reward = rewards[i]
states = torch.tensor(self.states, dtype=torch.float)
actions = torch.tensor(self.actions, dtype=torch.long).view(-1, 1)
all_old_probs = torch.tensor(self.a_probs,
dtype=torch.float).view(-1, 1)
rewards = rewards.reshape(-1, 1)
actor_entropies = []
actor_losses = []
for batch in BatchSampler(SubsetRandomSampler(range(len(self.states))),
batch_size,
drop_last=False):
states_batch = states[batch].numpy()
actions_batch = actions[batch].numpy()
old_action_probs = all_old_probs[batch].numpy()
rewards_batch = rewards[batch]
actor_loss = []
actor_entropy = []
for state, action, reward, old_action_probs in zip(
states_batch, actions_batch, rewards_batch,
old_action_probs):
state = state.reshape((-1, 1))
V = self.critic.forward(state)
advantage = reward - V
probs = self.actor.forward(state)
actor_entropy.append(-np.sum(np.log2(probs) * probs))
action_prob = probs[action]
action_prob_ratio = action_prob / old_action_probs
surr = action_prob_ratio * advantage
if action_prob_ratio < (1 - self.clip_e):
clamp_prob = 1 - self.clip_e
elif action_prob_ratio > (1 + self.clip_e):
clamp_prob = 1 + self.clip_e
else:
clamp_prob = action_prob_ratio
clipped_surr = clamp_prob * advantage
actor_loss.append(-np.minimum(surr, clipped_surr))
if surr < clipped_surr:
actor_dLoss = 1 / old_action_probs * advantage
else:
if action_prob_ratio < 1 - self.clip_e or action_prob_ratio > 1 + self.clip_e:
actor_dLoss = 0
else:
actor_dLoss = 1 / old_action_probs * advantage
self.actor.backward(actor_dLoss, action)
critic_loss = np.square(advantage)
critic_dloss = 2 * advantage
self.critic.backward(critic_dloss, action)
self.actor.mean_grads(batch_size)
self.actor.update()
actor_losses.append(np.mean(actor_loss))
actor_entropies.append(np.mean(actor_entropy))
self.critic.mean_grads(batch_size)
self.critic.update()
self._clear_buffers()
return np.mean(actor_losses), np.mean(actor_entropies), self.a_lr
#DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'
DEVICE = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
collate_fn = BertCollator(device='cpu')
if __name__ == '__main__':
dataset = AmazonZiser17(ds=SOURCE, dl=0, labeled=True, cldata=False)
dataset_size = len(dataset)
indices = list(range(dataset_size))
perm = torch.randperm(len(indices))
val_size = 0.2
val_split = int(np.floor(val_size * dataset_size))
train_indices = perm[val_split:]
val_indices = perm[:val_split]
train_sampler = SubsetRandomSampler(train_indices)
val_sampler = SubsetRandomSampler(val_indices)
train_loader = DataLoader(dataset,
batch_size=4,
sampler=train_sampler,
drop_last=False,
collate_fn=collate_fn)
val_loader = DataLoader(dataset,
batch_size=4,
sampler=val_sampler,
drop_last=False,
collate_fn=collate_fn)
if TARGET == "books":
pre = './sbooks'
elif TARGET == "dvd":
def train_val_split(
dataset: Dataset,
batch_size: int,
val_split: float,
shuffle: bool = True,
num_workers: int = 4,
collate_fn: Callable = None,
pin_memory: bool = True,
drop_last: bool = False,
timeout: float = 0,
worker_init_fn: Callable = None,
val_transform: Callable = None,
target_val_transform: Callable = None,
use_default_val_transform: bool = False,
) -> Tuple[DataLoader, DataLoader]:
"""
Splits the dataset into train and validation dataloaders.
Args:
batch_size (int): The batch size
dataset (Dataset): The dataset to split
val_split (float): The amount of data from the original dataset to be used \
in the validation dataloader. Should be a value between 0 and 1.
shuffle (bool, optional): If ``True``, the train and validation data are chosen at \
random. Defaults to ``True``.
num_workers (int, optional): How many subprocesses to use for data loading. Defaults to 4.
collate_fn (Callable, optional): Merges a list of samples to form a mini-batch of Tensor(s). \
Used when using batched loading from a map-style dataset. Defaults to ``None``.
pin_memory (bool, optional): If ``True``, copies Tensors to cuda pinned memory. Defaults to ``True``.
drop_last (bool, optional): Set to ``True`` to drop the last incomplete batch. Defaults to ``False``.
timeout (float, optional): If positive, the timeout value for collecting a batch from workers. \
Should always be non-negative. Defaults to 0.
worker_init_fn (Callable, optional): If not None, this will be called on each worker subprocess with \
the worker id (an int in [0, num_workers - 1]) as input, after seeding and before data loading.
val_transform (Callable, optional): The transform to be used in the val dataset. Defaults to None.
target_val_transform (Callable, optional): The transform to be used for the targets in the val dataset. \
Defaults to None.
use_default_val_transform (bool, optional): Whether to use the default val transforms or not. \
Defaults to False.
Returns:
Tuple[DataLoader, DataLoader]: The train dataloader and the validation dataloader
"""
# getting dataset size
dataset_size = len(dataset)
# generating indices
indices = np.arange(dataset_size)
if shuffle:
np.random.shuffle(indices)
# getting amount of data in validation split
split = np.floor(val_split * dataset_size).astype(np.int)
# generating samplers
train_sampler = SubsetRandomSampler(indices[split:])
val_sampler = SubsetRandomSampler(indices[:split])
# creating val dataset
val_dataset = deepcopy(dataset)
# setting the transform for the val dataset
if use_default_val_transform:
val_transform = val_dataset.default_val_transform()
val_dataset.train = False
val_dataset.transform = val_transform
val_dataset.target_transform = target_val_transform
# creating and returning dataloaders
return (
DataLoader(
dataset,
batch_size=batch_size,
sampler=train_sampler,
num_workers=num_workers,
collate_fn=collate_fn,
pin_memory=pin_memory,
drop_last=drop_last,
timeout=timeout,
worker_init_fn=worker_init_fn,
),
DataLoader(
val_dataset,
batch_size=batch_size,
sampler=val_sampler,
num_workers=num_workers,
collate_fn=collate_fn,
pin_memory=pin_memory,
drop_last=drop_last,
timeout=timeout,
worker_init_fn=worker_init_fn,
),
)
def main():
parsed_args = parse_args()
init_logging(parsed_args.logger)
# dataset related
davis_dir = parsed_args.dataset
year = parsed_args.year
dataset_mode = parsed_args.set
seq_names = parsed_args.sequences
shuffle = not parsed_args.no_shuffle
# model related
mode = parsed_args.mode
cuda_dev = parsed_args.cuda_device
model_save_path = parsed_args.model_save
model_load_path = parsed_args.model_load
# training related
batch_size = parsed_args.batch_size
epochs = parsed_args.epochs
iters = parsed_args.iters
augment = not parsed_args.no_augment
lr = parsed_args.learning_rate
weight_decay = parsed_args.weight_decay
validation_size = parsed_args.validation_size
loss_function = parsed_args.loss_function
# videomatch related
img_shape = parsed_args.input_image_shape
seg_shape = parsed_args.segmentation_shape
remove_outliers = not parsed_args.leave_outliers
fg_thresh = parsed_args.fg_thresh
encoder = parsed_args.encoder
upsample_fac = parsed_args.upsample_factor
# misc
val_report_iter = parsed_args.val_report
visualize = parsed_args.visualize
results_dir = parsed_args.results_dir
loss_visualize = parsed_args.loss_visualization
# args checks
if mode == 'train' and batch_size != 1:
logger.warning("Batch size > 1 is only applicable to 'eval' mode.")
if iters != -1 and epochs > 1:
logger.warning("Iters is set to {} and not to -1 (full dataset), but epoch is > 1".format(iters))
if mode == 'eval' and shuffle:
logger.warning("Dataset shuffle can't be set to True in 'eval' mode, setting it to False!")
shuffle = False
if mode == 'train' and not shuffle:
logger.warning("Dataset shuffle is off, consider turning it on when training, "
"to avoid overfitting on starting sequences")
if mode != 'eval' and visualize:
logger.warning("Visualize is set to True, but mode isn't 'eval'")
device = None if cuda_dev is None else "cuda:{:d}".format(cuda_dev)
dataset = Davis(davis_dir, dataset_mode, seq_names)
vm = VideoMatch(out_shape=seg_shape, device=device, encoder=encoder, upsample_fac=upsample_fac)
if model_load_path is not None:
logger.info("Loading model from path {}".format(model_load_path))
vm.load_model(model_load_path)
if mode == 'train':
pair_sampler = PairSampler(dataset, randomize=shuffle)
indices = np.arange(len(pair_sampler))
if shuffle:
np.random.shuffle(indices)
split = int(np.floor(validation_size * len(pair_sampler)))
val_indices = indices[:split]
train_indices = indices[split:]
train_loader = DataLoader(dataset, batch_sampler=SubsetRandomSampler(pair_sampler.get_indexes(train_indices)),
collate_fn=collate_pairs)
val_loader = DataLoader(dataset, batch_sampler=SubsetRandomSampler(pair_sampler.get_indexes(val_indices)),
collate_fn=collate_pairs)
logger.debug("Train set size: {}, Validation set size: {}".format(len(pair_sampler) - split, split))
iters = len(train_loader) if iters == -1 else iters
fp = FrameAugmentor(img_shape, augment)
train_vm(train_loader, val_loader, vm, fp, device, lr, weight_decay, iters, epochs,
val_report_iter, model_save_path, loss_visualize, loss_function)
elif mode == 'eval':
multiframe_sampler = MultiFrameSampler(dataset)
data_loader = DataLoader(dataset, sampler=multiframe_sampler, collate_fn=collate_multiframes,
batch_size=batch_size, num_workers=batch_size)
if results_dir is not None and not isdir(results_dir):
mkdir(results_dir)
eval_vm(data_loader, vm, img_shape, fg_thresh, remove_outliers, visualize, results_dir)
kwargs['node_feature_func'] = 'property_prediction'
kwargs['edge_feature_func'] = 'default'
if args.task in ['hiv']:
kwargs['k_fold'] = 'StratifiedKFold'
else:
kwargs['k_fold'] = 'KFold'
dataset = config_task2dataset[args.task](**kwargs)
train_indices, test_indices = split_into_KFold(dataset=dataset,
k=args.k_fold,
index=args.running_index,
**kwargs)
train_sampler = SubsetRandomSampler(train_indices)
test_sampler = SubsetRandomSampler(test_indices)
train_dataloader = DataLoader(dataset,
sampler=train_sampler,
batch_size=args.batch_size)
test_dataloader = DataLoader(dataset,
sampler=test_sampler,
batch_size=args.batch_size)
if args.model == 'ECFP':
model = config_model[args.model](ECFP_dim=args.fp_length,
hidden_dim=args.fp_hiddden_dim,
output_dim=1)
elif args.model == 'GIN':
model = config_model[args.model](dataset.node_feature_dim,
[256, 256, 256])
def train(self, batch_size=128):
# PPO algorithm
# Unroll rewards
rewards = np.array(self.rewards)
reward = 0
for i in reversed(range(len(self.rewards))):
rewards[i] += self.gamma * reward
reward = rewards[i]
states = torch.tensor(self.states, dtype=torch.float)
actions = torch.tensor(self.actions,
dtype=torch.float).view(-1, self.action_space)
all_old_probs = torch.tensor(self.a_probs, dtype=torch.float).view(
-1, self.action_space)
rewards = rewards.reshape(-1, 1)
actor_entropies = []
actor_losses = []
for batch in BatchSampler(SubsetRandomSampler(range(len(self.states))),
batch_size,
drop_last=False):
states_batch = states[batch].numpy()
actions_batch = actions[batch].numpy()
old_action_probs = all_old_probs[batch].numpy()
rewards_batch = rewards[batch]
actor_grads_batch = []
critic_grads_batch = []
actor_loss = []
actor_entropy = []
for state, action, reward, old_action_probs in zip(
states_batch, actions_batch, rewards_batch,
old_action_probs):
state = state.reshape(
(-1,
1)) # TODO maybe check shape? Does not coincide with act
V, critic_cache = self.critic.full_forward_propagation(state)
advantage = reward - V
_, action_prob, entropy, actor_cache = self.act(state)
actor_entropy.append(np.sum(entropy.numpy()))
old_action_probs = old_action_probs.reshape(-1, 1)
action_prob_ratio = action_prob / old_action_probs
surr = action_prob_ratio * advantage
clamp_prob = action_prob_ratio.copy()
clamp_prob[action_prob_ratio < (1 -
self.clip_e)] = 1 - self.clip_e
clamp_prob[action_prob_ratio > (1 +
self.clip_e)] = 1 + self.clip_e
clipped_surr = clamp_prob * advantage
actor_loss.append(-np.minimum(surr, clipped_surr))
actor_dLoss = np.zeros(shape=(self.action_space, 1))
for i in range(self.action_space):
if surr[i] < clipped_surr[i]:
actor_dLoss[i, 0] = 1 / old_action_probs[i] * advantage
else:
if action_prob_ratio[
i] < 1 - self.clip_e or action_prob_ratio[
i] > 1 + self.clip_e:
actor_dLoss[i, 0] = 0
else:
actor_dLoss[
i, 0] = 1 / old_action_probs[i] * advantage
actor_grads = self.actor.full_backward_propagation(
actor_dLoss, actor_cache, action)
actor_grads_batch.append(actor_grads)
critic_loss = np.square(advantage)
critic_dloss = 2 * advantage
critic_grads = self.critic.full_backward_propagation(
critic_dloss, critic_cache, action)
critic_grads_batch.append(critic_grads)
actor_grads_values_mean = self.actor.mean_grads(
actor_grads_batch, batch_size)
self.actor.update(actor_grads_values_mean, self.a_lr)
actor_losses.append(np.mean(actor_loss))
actor_entropies.append(np.mean(actor_entropy))
critic_grads_values_mean = self.critic.mean_grads(
critic_grads_batch, batch_size)
self.critic.update(critic_grads_values_mean, self.c_lr)
self._clear_buffers()
return np.mean(actor_losses), np.mean(actor_entropies), self.a_lr
def get_dataloader(module_name, module_args):
if module_args['dataset']['img_type'] == 'cv':
from opencv_transforms import opencv_transforms as transforms
else:
from torchvision import transforms
train_transfroms = transforms.Compose([
transforms.ColorJitter(brightness=0.5),
transforms.ToTensor()
])
val_transfroms = transforms.ToTensor()
# 创建数据集
dataset_args = module_args['dataset']
train_data_path = dataset_args.pop('train_data_path')
val_data_path = dataset_args.pop('val_data_path')
train_dataset = get_dataset(data_path=train_data_path,
module_name=module_name,
transform=train_transfroms,
phase='train',
dataset_args=dataset_args)
val_dataset = get_dataset(data_path=val_data_path,
module_name=module_name,
transform=val_transfroms,
phase='test',
dataset_args=dataset_args)
if val_dataset is None:
val_dataset = copy.deepcopy(train_dataset)
val_dataset.transform = transforms.ToTensor()
val_dataset.phase = 'test'
# 数据集切分
indices = list(range(len(train_dataset)))
random.shuffle(indices)
val_len = int(module_args['loader']['validation_split'] * len(train_dataset))
train_sampler = SubsetRandomSampler(indices[val_len:])
val_sampler = SubsetRandomSampler(indices[:val_len])
train_loader = DataLoader(dataset=train_dataset,
sampler=train_sampler,
batch_size=module_args['loader']['train_batch_size'],
pin_memory=module_args['loader']['pin_memory'],
num_workers=module_args['loader']['num_workers'])
val_loader = DataLoader(dataset=val_dataset,
sampler=val_sampler,
batch_size=module_args['loader']['val_batch_size'],
pin_memory=module_args['loader']['pin_memory'],
num_workers=module_args['loader']['num_workers'])
else:
train_loader = DataLoader(dataset=train_dataset,
batch_size=module_args['loader']['train_batch_size'],
shuffle=module_args['loader']['shuffle'],
pin_memory=module_args['loader']['pin_memory'],
num_workers=module_args['loader']['num_workers'])
val_loader = DataLoader(dataset=val_dataset,
batch_size=module_args['loader']['val_batch_size'],
shuffle=module_args['loader']['shuffle'],
pin_memory=module_args['loader']['pin_memory'],
num_workers=module_args['loader']['num_workers'])
return train_loader, val_loader
def get_clusters(cluster_set):
nDescriptors = 50000
nPerImage = 100
nIm = ceil(nDescriptors / nPerImage)
sampler = SubsetRandomSampler(
np.random.choice(len(cluster_set), nIm, replace=False))
data_loader = DataLoader(dataset=cluster_set,
num_workers=opt.threads,
batch_size=opt.cacheBatchSize,
shuffle=False,
pin_memory=cuda,
sampler=sampler)
if not exists(join(opt.dataPath, 'centroids')):
makedirs(join(opt.dataPath, 'centroids'))
initcache = join(
opt.dataPath, 'centroids', opt.arch + '_' + cluster_set.dataset + '_' +
str(opt.num_clusters) + '_desc_cen.hdf5')
with h5py.File(initcache, mode='w') as h5:
with torch.no_grad():
model.eval()
print('====> Extracting Descriptors')
dbFeat = h5.create_dataset("descriptors",
[nDescriptors, encoder_dim],
dtype=np.float32)
for iteration, (input, indices) in enumerate(data_loader, 1):
input = input.to(device)
print(input.size()) #torch.Size([8, 3, 480, 640])
image_descriptors = model.encoder(input).view(
input.size(0), encoder_dim, -1).permute(0, 2, 1)
print(image_descriptors.shape) #torch.Size([8, 1200, 512])
print(image_descriptors.size(1)) #1200
batchix = (iteration - 1) * opt.cacheBatchSize * nPerImage
for ix in range(image_descriptors.size(0)):
# sample different location for each image in batch
sample = np.random.choice(image_descriptors.size(1),
nPerImage,
replace=False)
startix = batchix + ix * nPerImage
dbFeat[startix:startix + nPerImage, :] = image_descriptors[
ix, sample, :].detach().cpu().numpy()
if iteration % 50 == 0 or len(data_loader) <= 10:
print("==> Batch ({}/{})".format(
iteration, ceil(nIm / opt.cacheBatchSize)),
flush=True)
del input, image_descriptors
print('====> Clustering..')
niter = 100
kmeans = faiss.Kmeans(encoder_dim,
opt.num_clusters,
niter=niter,
verbose=False)
kmeans.train(dbFeat[...])
print('====> Storing centroids', kmeans.centroids.shape)
h5.create_dataset('centroids', data=kmeans.centroids)
print('====> Done!')
def main(options):
model_dir = os.path.join('experiments', 'models')
trg_size = (224, 224)
transformations = transforms.Compose([CustomResize(trg_size),
CustomToTensor()
])
batch_size = options.batch_size
train_set = SliceSet2D(transform=transformations)
validation_split = .4
shuffle_dataset = True
random_seed = 42
set_size = len(train_set)
indices = list(range(set_size))
split = int(np.floor(validation_split * set_size))
if shuffle_dataset:
np.random.seed(random_seed)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
print("Set Size: {}|Train Size: {}| Validation Size: {}".format(len(indices), len(train_indices), len(val_indices)))
print("Model will be saved in \" ./experiments/model\". Start training...")
# Creating data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size,
sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size,
sampler=valid_sampler)
# Initiate the model
model = models.resnet18(pretrained=True)
model.fc = Linear(in_features=512, out_features=3)
for para in list(model.parameters())[:-2]:
para.requires_grad = False
optimizer = Adam(params=[model.fc.weight, model.fc.bias], lr=3e-4)
print('the training layer is:')
for name, param in model.named_parameters(): # 查看可优化的参数有哪些
if param.requires_grad:
print(name)
# model.load_state_dict(torch.load(options.load))
if torch.cuda.is_available():
model.cuda()
else:
model.cpu()
# Binary cross-entropy loss
criterion = torch.nn.CrossEntropyLoss()
# criterion = torch.nn.CrossEntropyLoss()
lr = options.learning_rate
optimizer = eval("torch.optim." + options.optimizer)(filter(lambda x: x.requires_grad, model.parameters()), lr)
best_accuracy = float("-inf")
for epoch_i in range(options.epochs):
logging.info("At {0}-th epoch.".format(epoch_i))
train_loss, correct_cnt = train(model, train_loader, criterion, optimizer)
# each instance in one batch has 3 views
train_avg_loss = train_loss / len(train_indices)*3/options.batch_size
train_avg_acu = float(correct_cnt) / (len(train_indices) * 3)
logging.info(
"Average training loss is {0:.5f} at the end of epoch {1}".format(train_avg_loss.data, epoch_i))
logging.info("Average training accuracy is {0:.5f} at the end of epoch {1}".format(train_avg_acu, epoch_i))
with torch.no_grad():
correct_cnt = validate(model, validation_loader)
dev_avg_acu = float(correct_cnt) / len(val_indices)
logging.info("Average validation accuracy is {0:.5f} at the end of epoch {1}".format(dev_avg_acu, epoch_i))
if dev_avg_acu > best_accuracy:
best_accuracy = dev_avg_acu
torch.save(model, os.path.join('experiments', 'models/mymodel.pth'))
