def run(self, task_func=None, interval=1.0):
# set task attributes
self.task_func = task_func
self.interval = interval
# run model while replacing task function with wrapper
BasicModel.run(self, main_task_func_wrapper)
def run(self, task_func=None, interval=1.0):
# set task attributes
self.task_func = task_func
self.interval = interval
# run model while replacing task function with wrapper
BasicModel.run(self, main_task_func_wrapper)
def __init__(
self, logger,
channel=None,
parent=None, suppress_exc=True,
worker_cls=None, table_cls=None):
BasicModel.__init__(
self, logger, parent, suppress_exc,
worker_cls, table_cls)
self.channel = channel or QueueChannel(maxsize=100)
def __init__(self,
logger,
channel=None,
parent=None,
suppress_exc=True,
worker_cls=None,
table_cls=None):
BasicModel.__init__(self, logger, parent, suppress_exc, worker_cls,
table_cls)
self.channel = channel or QueueChannel(maxsize=100)
def main():
# Step 0: preparation
place = paddle.fluid.CUDAPlace(0)
with fluid.dygraph.guard(place):
# Step 1: Define training dataloader
basic_augmentation = TrainAugmentation(image_size=256)
basic_dataloader = BasicDataLoader(image_folder=args.image_folder,
image_list_file=args.image_list_file,
transform=basic_augmentation,
shuffle=True)
train_dataloader = fluid.io.DataLoader.from_generator(capacity=10, use_multiprocess=True)
train_dataloader.set_sample_generator(basic_dataloader, batch_size=args.batch_size, places=place)
total_batch = int(len(basic_dataloader) / args.batch_size)
# Step 2: Create model
if args.net == "basic":
model = BasicModel()
else:
raise NotImplementedError(f"args.net: {args.net} is not Supported!")
# Step 3: Define criterion and optimizer
criterion = Basic_SegLoss
optimizer = AdamOptimizer(learning_rate=args.lr, parameter_list=model.parameters())
# create optimizer
# Step 4: Training
for epoch in range(1, args.num_epochs+1):
train_loss = train(train_dataloader,
model,
criterion,
optimizer,
epoch,
total_batch)
print(f"----- Epoch[{epoch}/{args.num_epochs}] Train Loss: {train_loss:.4f}")
if epoch % args.save_freq == 0 or epoch == args.num_epochs:
model_path = os.path.join(args.checkpoint_folder, f"{args.net}-Epoch-{epoch}-Loss-{train_loss}")
# TODO: save model and optmizer states
model_dict = model.state_dict()
fluid.save_dygraph(model_dict, model_path)
optimizer_dict = optimizer.state_dict()
fluid.save_dygraph(optimizer_dict, model_path)
print(f'----- Save model: {model_path}.pdparams')
print(f'----- Save optimizer: {model_path}.pdopt')
def __init__(self, config):
BasicModel.__init__(self, config)
self.channels = self.config.get('channels', None)
if self.channels is None:
raise ValueError(
"Error: No channels for convolutions defined. "
"These values have to be specified for all CNN architectures."
)
if len(self.channels) != self.nb_layers + 1:
raise ValueError(
"Error: Invalid number of specified convolution channels. "
"The length of channels should equal nb_layers + 1."
)
self.image_size = self.config.get('image_size', None)
if self.image_size is None:
raise ValueError(
"Error: No image_size specified. "
"This is mandatory for using CNN architectures."
)
if np.shape(self.image_size) != (2,):
raise ValueError(
"Error: Invalid shape of image_size. "
"The provided shape should be (2,)."
)
self.fc_units = []
self.fc_units.append(self.image_size[0] * self.image_size[1] * self.channels[-1])
if isinstance(self.nb_units, int) and self.nb_units != 0:
self.fc_units.append(self.nb_units)
elif isinstance(self.nb_units, (list, np.ndarray)):
for unit_value in self.nb_units:
self.fc_units.append(unit_value)
self.fc_units.append(self.output_size)
self.conv_layer
self.fc_layer
def register_workers(
self, group,
job_func, init_func=None, term_func=None,
num=1):
# create workers while replacing task function with wrapper
workers = BasicModel.register_workers(
self, group, worker_task_func_wrapper, num)
# set worker attributes
for worker in workers:
worker.channel = self.channel
worker.job_func = job_func
worker.init_func = init_func
worker.term_func = term_func
return workers
def register_workers(self,
group,
job_func,
init_func=None,
term_func=None,
num=1):
# create workers while replacing task function with wrapper
workers = BasicModel.register_workers(self, group,
worker_task_func_wrapper, num)
# set worker attributes
for worker in workers:
worker.channel = self.channel
worker.job_func = job_func
worker.init_func = init_func
worker.term_func = term_func
return workers
def main(args):
# 0. env preparation
with fluid.dygraph.guard(fluid.CPUPlace()):
# 1. create model
model = BasicModel()
model.eval()
# 2. load pretrained model
params_dict, _ = fluid.load_dygraph(args.model_path)
model.load_dict(params_dict)
# 3. read test image list
data_list = []
with open(args.imagelist, 'r') as infile:
for line in infile:
data_path = os.path.join(args.image_folder, line.split()[0])
label_path = os.path.join(args.image_folder, line.split()[1])
data_list.append((data_path, label_path))
# 4. create transforms for test image, transform should be same as training
transforms = InferAugmentation()
color_list = []
with open('pascal_context_colors.txt', 'r') as colorfile:
for line in infile:
ll = line.split()
color_list.append(int(ll[0]))
color_list.append(int(ll[1]))
color_list.append(int(ll[2]))
# 5. loop over list of images
for index, data in enumerate(data_list):
# 6. read image and do preprocessing
image = cv2.imread(data[0], cv2.IMREAD_COLOR)
image = cv2.cvtColor(data, cv2.COLOR)BGR2RGB
image = image[np.newaxis,:,:,:]
# 7. image to variable
image = fluid.dygraph.to_variable(image)
image = fluid.layers.transpose(image, [0, 3, 1, 2])
# 8. call inference func
image = model(image)
# 9. save results
image = fluid.layers.squeeze(image)
pred = colorize(image.numpy() ,color_list)
cv2.imwrite(str(index)+ '_.png', pred)
save_blend_image(data[0], 'tmp.png')
def train():
trainloader, testloader = make_dataloader()
# build model
model = BasicModel()
# loss func
loss_func = nn.CrossEntropyLoss()
# optimzier
optimizier = optim.Adam(model.parameters(), lr=1e-3)
# configuration
epochs = 10
# training
for epoch in range(epochs):
model.train()
pbar = tqdm(trainloader)
for image, label in pbar:
# forward
output = model(image)
# compute loss
loss = loss_func(output, label)
optimizier.zero_grad()
loss.backward()
optimizier.step()
# compute batch accuracy
predicts = torch.argmax(output, dim=-1)
accu = torch.sum(predicts == label).float() / image.size(0)
pbar.set_description('Epoch:[{:02d}]-Loss:{:.3f}-Accu:{:.3f}'\
.format(epoch+1,loss.item(),accu.item()))
# testing
model.eval()
with torch.no_grad():
corrects = 0
total_nums = 0
for image, label in tqdm(testloader):
output = model(image)
predicts = torch.argmax(output, dim=-1)
corrects += (predicts == label).sum()
total_nums += label.size(0)
test_accu = corrects.float() / total_nums
print('Epoch:[{:02d}]-Test_Accu:{:.3f}'.format(
epoch + 1, test_accu.item()))
def run(self, task_func=None):
try:
BasicModel.run(self, task_func)
finally:
self.channel.clear()
def __init__(self, config):
BasicModel.__init__(self, config)
self.input_layer
self.hidden_layer
self.output_layer
def evaluate(model_class):
y_true = df_val[['nose_scaled_x', 'nose_scaled_y']]
model = model_class.get_model()
custom_generator = model_class.modify_generator(val_generator)
y_model = model.predict(custom_generator, steps=np.ceil(len(df_val) / 16))
y_model = model_class.transform_prediction(y_model)[:len(y_true)]
mse = mean_squared_error(y_true, y_model)
print("model loss: ", mse)
return mse
if __name__ == "__main__":
args = parser.parse_args()
try:
if args.model == 'basic':
model_class = BasicModel()
elif args.model == 'pyramid':
model_class = PyramidModel()
else:
raise Exception("available model types: basic, pyramid")
model_class.load_model(args.path)
evaluate(model_class)
except Exception as e:
print(e)
train_generator.reset()
val_generator.reset()
for layer in backbone_model.layers[-20:]:
layer.trainable = True
model.fit_generator(custom_train_generator,
validation_data=custom_val_generator,
validation_steps=50,
steps_per_epoch=100,
epochs=30,
callbacks=callbacks)
model.save(model_path)
if __name__ == "__main__":
args = parser.parse_args()
try:
if args.model == 'basic':
model_class = BasicModel()
elif args.model == 'pyramid':
model_class = PyramidModel()
else:
raise Exception("available model types: basic, pyramid")
train(model_class, args.path)
except Exception as e:
print(e)
def __init__(self, config, input_batches_file, is_training):
BasicModel.__init__(self, config, input_batches_file, is_training)
def format_monitor(self):
string = BasicModel.format_monitor(self)
string = format_channel(self.channel, 'Jobs', string)
return string
for batch in range(int(number_of_images / batch_size)):
counter += 1
images, labels = tools.batch_dispatch()
if images is None:
break
loss, summary = session.run([cost, merged],
feed_dict={
images_ph: images,
labels_ph: labels
})
print('loss', loss)
session.run(optimizer,
feed_dict={
images_ph: images,
labels_ph: labels
})
print('Epoch number ', epoch, 'batch', batch, 'complete')
writer.add_summary(summary, counter)
saver.save(session, model_save_name)
if __name__ == '__main__':
tools = Utils()
model = BasicModel()
network = model_architecture.generate_model(images_ph, number_of_classes)
print('Training started')
print(network)
number_of_images = sum([len(files) for r, d, files in os.walk('data')])
trainer(network, number_of_images)
def run(self, task_func=None):
try:
BasicModel.run(self, task_func)
finally:
self.channel.clear()
def format_monitor(self):
string = BasicModel.format_monitor(self)
string = format_channel(self.channel, 'Jobs', string)
return string