def train_single(cw):
c, w = cw
data = PennFudanDataset('dataset/PennFudanPed')
data.samples = data.samples[:30]
process(data, create_pipeline(threshold=c))
inputs, targets = extractor.extract(data, w=w, N=100000, threaded=False)
score = train_model(inputs, targets)
print('c={0} w={1} p={2:.4f} r={3:.4f} f1={4:.4f}'.format(c, w, *score))
return (cw, score)
def train_single(cw):
c, w = cw
data = PennFudanDataset('dataset/PennFudanPed')
data.samples = data.samples[:30]
process(data, create_pipeline(threshold=c))
inputs, targets = extractor.extract(data, w=w, N=100000, threaded=False)
score = train_model(inputs, targets)
print('c={0} w={1} p={2:.4f} r={3:.4f} f1={4:.4f}'.format(c, w, *score))
return (cw, score)
model = GridSearchCV(estimator, grid, n_jobs=threads, verbose=verbose)
model.fit(inputs, targets)
return model.best_estimator_
def generate_sets(dataset, w=11, N=5000):
process(dataset, create_pipeline())
return extractor.extract(dataset, w=w, N=N)
if __name__ == '__main__':
grid = {
'kernel': ['rbf'],
'C': [2e-3, 2e-1, 1, 2e1, 2e3],
'gamma': [2e-7, 2e-3, 2e-1, 1, 2e1]
}
dataset = PennFudanDataset('dataset/PennFudanPed')
# generate train and test sets
inputs, targets = generate_sets(dataset)
# search for the best estimator using grid search
best = grid_search(grid, inputs, targets)
# print the params of the best estimator found using grid search
logger.info(best)
# dump the best estimator to file for further use
joblib.dump(best, 'grid_best.pkl')
import torch
from models.MaskRCNN import get_model_instance_segmentation
from dataset import PennFudanDataset, get_transform
from references.engine import train_one_epoch, evaluate
from references import utils
# train on the GPU or the CPU, if a GPU is not available
device = torch.device('cuda') if torch.cuda.is_available() else torch.device(
'cpu')
# our dataset has two classes only - background and person
num_classes = 2
# use out dataset an defined transformations
dataset = PennFudanDataset('PennFudanPed', get_transform(train=True))
dataset_test = PennFudanDataset('PennFudanPed', get_transform(train=False))
# split the dataset in train and test set
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=2,
shuffle=True,
num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
def main():
global hparams, args
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device(
hparams.device) if torch.cuda.is_available() else torch.device('cpu')
if (args.dataset == 'malaria'):
hparams.dataset_root = 'malaria'
hparams.exp_name = f'maskrcnn-{hparams.dataset_root}'
dataset = MalariaDataset(hparams.train_dir, hparams.train_csv,
get_transform(train=True))
dataset_test = MalariaDataset(hparams.test_dir, hparams.test_csv,
get_transform(False))
else:
dataset = PennFudanDataset('PennFudanPed', get_transform(train=True))
dataset_test = PennFudanDataset('PennFudanPed',
get_transform(train=False))
hparams.num_classes = 2
writer = SummaryWriter(f'runs/{hparams.exp_name}_{hparams.timestamp}')
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=2,
shuffle=True,
num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
num_workers=4,
collate_fn=utils.collate_fn)
# get the model using our helper function
model = get_model_instance_segmentation(hparams)
# move model to the right device
model.to(device)
model_without_ddp = model
if hparams.distributed:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[hparams.device_ids])
model_without_ddp = model.module
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params,
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
# let's train it for 10 epochs
num_epochs = hparams.num_epochs
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model,
optimizer,
data_loader,
device,
epoch,
print_freq=10,
writer=writer)
# update the learning rate
lr_scheduler.step()
# evaluate on the test dataset
evaluate(model, data_loader_test, device=device)
torch.save(
{
'model': model_without_ddp.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}, os.path.join(hparams.model_dir, 'model_{}.pth'.format(epoch)))
print("That's it!")
def main():
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
print(f'Device: {device}')
# our dataset has two classes only - background and person
num_classes = 2
# use our dataset and defined transformations
dataset = PennFudanDataset('PennFudanPed', get_transform(train=True))
dataset_test = PennFudanDataset('PennFudanPed', get_transform(train=False))
# split the dataset in train and test set
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=1,
shuffle=True,
num_workers=1,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
num_workers=1,
collate_fn=utils.collate_fn)
# get the model using our helper function
model = get_model_instance_segmentation(num_classes)
# move model to the right device
model.to(device)
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params,
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
# let's train it for 10 epochs
num_epochs = 10
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model,
optimizer,
data_loader,
device,
epoch,
print_freq=10)
# update the learning rate
lr_scheduler.step()
# evaluate on the test dataset
evaluate(model, data_loader_test, device=device)
print("That's it!")
def main(root,
num_classes,
num_epochs,
batch_size,
label_mapping_dict={},
backbone=None,
save_model_path="./trained_model.pth",
save_checkpoints=False,
ignore_list=[],
data="custom",
load_model_path=""):
"""
Arguments:
load_model_path (string): load model from given path for retraining.
Known Issue:
for retraining model, currently need load whole model. state dict and optimzer state restoration not yet supported.
TODO: Add load state dict support.
TODO: Add Optimizer state resume support
"""
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# use our dataset and defined transformations
#dataset = PascalVocDataset(root, get_transform(train=True), data)
#dataset_test = PascalVocDataset(root, get_transform(train=False), data)
if data == "ped":
dataset = PennFudanDataset(root, get_transform(train=True))
dataset_test = PennFudanDataset(root, get_transform(train=False))
elif data == "custom":
dataset = CustomDataset(root, get_transform(train=True),
label_mapping_dict, ignore_list)
dataset_test = CustomDataset(root, get_transform(train=False),
label_mapping_dict, ignore_list)
else:
dataset = PascalVocDataset(root, get_transform(train=True),
label_mapping_dict)
dataset_test = PascalVocDataset(root, get_transform(train=False),
label_mapping_dict)
print("preparing dataset....")
# split the dataset in train and test set
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])
print("Done.")
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
collate_fn=utils.collate_fn)
if not load_model_path:
# get the model using our helper function
model = get_model(num_classes, backbone)
else:
if os.path.exists(load_model_path):
model = torch.load(load_model_path)
else:
print("model path: {} does not exists".format(load_model_path))
# move model to the right device
model.to(device)
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params,
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
if not os.path.exists("./checkpoints"):
os.makedirs("./checkpoints")
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model,
optimizer,
data_loader,
device,
epoch,
print_freq=10)
# update the learning rate
lr_scheduler.step()
# evaluate on the test dataset
evaluate(model, data_loader_test, device=device)
if save_checkpoints:
#load model to cpu
save_model = model.to(torch.device("cpu"))
training_checkpoint_path = f"./checkpoints/training_checkpoint_{epoch}/{num_epochs}.pth"
#save model
torch.save(save_model, training_checkpoint_path)
#load model to cpu
model = model.to(torch.device("cpu"))
#save model
torch.save(model, save_model_path)
print("That's it!")
def main():
savePath = 'model.pth'
selectedDevice = ''
# let's train it for 10 epochs
num_epochs = 10
try:
opts, args = getopt.getopt(sys.argv[1:], "o:d:e:",
["modelOutputPath=", "device=", "epochs="])
except getopt.error as err:
print(str(err))
sys.exit(2)
for opt, arg in opts:
if opt in ("-o", "--modelOutputPath"):
if arg.endswith(".pth") is True or arg.endswith('.pt') is True:
savePath = arg
else:
print("Input Error: modelOutputPath must end in .pth or .pt")
sys.exit(2)
if opt in ("-d", "--device"):
lower = arg.lower()
if lower == "gpu" or lower == "cpu":
selectedDevice = arg.lower()
else:
print("Input Error: Device must be either 'gpu' or 'cpu'")
sys.exit(2)
if opt in ("-e", "--epochs"):
epochs = int(arg)
if epochs > 0:
num_epochs = epochs
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# check if user has requested either cpu or gpu processing
if torch.cuda.is_available() and selectedDevice == '':
print("CUDA device is detected, using GPU for training and evaluation")
elif selectedDevice != '':
if selectedDevice == 'gpu':
if torch.cuda.is_available() is False:
print("Cannot find CUDA driver or device")
sys.exit(2)
device = torch.device('cuda')
if selectedDevice == 'cpu':
device = torch.device('cpu')
# our dataset has two classes only - background and person
num_classes = 2
# use our dataset and defined transformations
dataset = PennFudanDataset('PennFudanPed', get_transform(train=True))
dataset_test = PennFudanDataset('PennFudanPed', get_transform(train=False))
# split the dataset in train and test
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-50])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:])
# define training and validation data loaders
# these organize the process for sending information to the GPU
data_loader = torch.utils.data.DataLoader(dataset,
batch_size=2,
shuffle=True,
num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
num_workers=4,
collate_fn=utils.collate_fn)
# get the model using our helper function
model = get_model_instance_segmentation(num_classes)
# move model to the selected device
model.to(device)
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params,
lr=0.005,
momentum=0.9,
weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=3,
gamma=0.1)
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
train_one_epoch(model,
optimizer,
data_loader,
device,
epoch,
print_freq=10)
# update the learning rate
lr_scheduler.step()
# evaluate on the test dataset
evaluate(model, data_loader_test, device=device)
# save the model to the current directory
torch.save(model.state_dict(), savePath)
print("Finished")
def main():
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# our dataset has two classes only - background and person
num_classes = 2
# get the model using our helper function
model = get_model_instance_segmentation(num_classes)
# load pretrain_dict
pretrain_dict = torch.load(
os.path.join("C:\\zhulei\\maskRcnn\\models", "_epoch-9.pth"))
model.load_state_dict(pretrain_dict)
# move model to the right device
model.to(device)
# use our dataset and defined transformations
dataset_test = PennFudanDataset('C:\\zhulei\\maskRcnn\\data\\test',
get_transform(train=False))
data_loader_test = torch.utils.data.DataLoader(dataset_test,
batch_size=1,
shuffle=False,
num_workers=1,
collate_fn=utils.collate_fn)
n_threads = torch.get_num_threads()
# FIXME remove this and make paste_masks_in_image run on the GPU
torch.set_num_threads(1)
cpu_device = torch.device("cpu")
model.eval()
metric_logger = utils.MetricLogger(delimiter=" ")
header = 'Test:'
coco = get_coco_api_from_dataset(data_loader_test.dataset)
iou_types = _get_iou_types(model)
coco_evaluator = CocoEvaluator(coco, iou_types)
for image, targets in metric_logger.log_every(data_loader_test, 100,
header):
image = list(img.to(device) for img in image)
targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
torch.cuda.synchronize()
model_time = time.time()
outputs = model(image)
instance_segmentation_api(image[0], outputs)
# 可视化
# for img in image:
# Image.fromarray((img.mul(255).permute(1, 2, 0).byte().cpu().numpy())[0])
# print(outputs[0]['masks'].shape)
# for i in range(99):
# result = Image.fromarray(outputs[0]['masks'][i, 0].mul(255).byte().cpu().numpy())
# result.show()
outputs = [{k: v.to(cpu_device)
for k, v in t.items()} for t in outputs]
model_time = time.time() - model_time
res = {
target["image_id"].item(): output
for target, output in zip(targets, outputs)
}
evaluator_time = time.time()
coco_evaluator.update(res)
evaluator_time = time.time() - evaluator_time
metric_logger.update(model_time=model_time,
evaluator_time=evaluator_time)
# gather the stats from all processes
metric_logger.synchronize_between_processes()
print("Averaged stats:", metric_logger)
coco_evaluator.synchronize_between_processes()
# accumulate predictions from all images
coco_evaluator.accumulate()
coco_evaluator.summarize()
torch.set_num_threads(n_threads)