def context(self, trainable=True, pretrained=True):
"""context for transfer learning.
Args:
trainable (bool): Set parameters in program to be trainable.
pretrained (bool) : Whether to load pretrained model.
Returns:
inputs (dict): key is 'image', corresponding vaule is image tensor.
outputs (dict): key is 'feature_map', corresponding value is the result of the layer before the fully connected layer.
context_prog (fluid.Program): program for transfer learning.
"""
context_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(context_prog, startup_prog):
with fluid.unique_name.guard():
image = fluid.layers.data(name="image",
shape=[3, 224, 224],
dtype="float32")
resnet_vd = ResNet50_vd()
feature_map = resnet_vd.net(input=image)
name_prefix = '@HUB_{}@'.format(self.name)
inputs = {'image': name_prefix + image.name}
outputs = {'feature_map': name_prefix + feature_map.name}
add_vars_prefix(context_prog, name_prefix)
add_vars_prefix(startup_prog, name_prefix)
global_vars = context_prog.global_block().vars
inputs = {
key: global_vars[value]
for key, value in inputs.items()
}
outputs = {
key: global_vars[value]
for key, value in outputs.items()
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# pretrained
if pretrained:
def _if_exist(var):
b = os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
return b
fluid.io.load_vars(exe,
self.default_pretrained_model_path,
context_prog,
predicate=_if_exist)
else:
exe.run(startup_prog)
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
def context(self,
body_feats,
yolo_head,
image,
trainable=True,
var_prefix=''):
"""Distill the Head Features, so as to perform transfer learning.
:param body_feats: feature maps of backbone
:type backbone: list
:param yolo_head: yolo_head of YOLOv3
:type yolo_head: <class 'YOLOv3Head' object>
:param image: image tensor.
:type image: <class 'paddle.fluid.framework.Variable'>
:param trainable: whether to set parameters trainable.
:type trainable: bool
:param var_prefix: the prefix of variables in yolo_head and backbone
:type var_prefix: str
"""
context_prog = image.block.program
with fluid.program_guard(context_prog):
im_size = fluid.layers.data(
name='im_size', shape=[2], dtype='int32')
head_features = yolo_head._get_outputs(
body_feats, is_train=trainable)
inputs = {
'image': var_prefix + image.name,
'im_size': var_prefix + im_size.name
}
# bbox_out = yolo_head.get_prediction(head_features, im_size)
outputs = {
'head_features':
[var_prefix + var.name for var in head_features]
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
inputs = {
key: context_prog.global_block().vars[value]
for key, value in inputs.items()
}
outputs = {
key: [
context_prog.global_block().vars[varname]
for varname in value
]
for key, value in outputs.items()
}
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
def context(self, trainable=False):
"""
Get the input ,output and program of the pretrained senta_bilstm
Args:
trainable(bool): whether fine-tune the pretrained parameters of senta_bilstm or not
Returns:
inputs(dict): the input variables of senta_bilstm (words)
outputs(dict): the output variables of senta_bilstm (the sentiment prediction results)
main_program(Program): the main_program of lac with pretrained prameters
"""
main_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
data = fluid.layers.data(name="words",
shape=[1],
dtype="int64",
lod_level=1)
data_name = data.name
pred, fc = bilstm_net(data, 1256606)
pred_name = pred.name
fc_name = fc.name
prefix_name = "@HUB_{}@".format(self.name)
add_vars_prefix(program=main_program, prefix=prefix_name)
for param in main_program.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# load the senta_lstm pretrained model
def if_exist(var):
return os.path.exists(
os.path.join(self.pretrained_model_path, var.name))
fluid.io.load_vars(exe,
self.pretrained_model_path,
predicate=if_exist)
inputs = {
"words":
main_program.global_block().vars[prefix_name + data_name]
}
outputs = {
"class_probs":
main_program.global_block().vars[prefix_name + pred_name],
"sentence_feature":
main_program.global_block().vars[prefix_name + fc_name]
}
return inputs, outputs, main_program
def context(self, trainable=False):
"""
Get the input ,output and program of the pretrained lac
Args:
trainable(bool): whether fine-tune the pretrained parameters of lac or not
Returns:
inputs(dict): the input variables of lac (words)
outputs(dict): the output variables of lac (the word segmentation results)
main_program(Program): the main_program of lac with pretrained prameters
"""
main_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
with fluid.unique_name.guard():
crf_decode, word, fc = lex_net(self.word_dict_len,
self.label_dict_len)
word_name = word.name
pred_name = crf_decode.name
fc_name = fc.name
prefix_name = "@HUB_{}@".format(self.name)
add_vars_prefix(program=main_program, prefix=prefix_name)
for param in main_program.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# load the lac pretrained model
def if_exist(var):
return os.path.exists(
os.path.join(self.pretrained_model_path, var.name))
fluid.io.load_vars(exe,
self.pretrained_model_path,
predicate=if_exist)
inputs = {
"words":
main_program.global_block().vars[prefix_name + word_name]
}
outputs = {
"predicted":
main_program.global_block().vars[prefix_name + pred_name],
"sentence_feature":
main_program.global_block().vars[prefix_name + fc_name]
}
return inputs, outputs, main_program
def context(self, trainable=True, pretrained=True, get_prediction=False):
"""
Distill the Head Features, so as to perform transfer learning.
Args:
trainable (bool): whether to set parameters trainable.
pretrained (bool): whether to load default pretrained model.
get_prediction (bool): whether to get prediction.
Returns:
inputs(dict): the input variables.
outputs(dict): the output variables.
context_prog (Program): the program to execute transfer learning.
"""
context_prog = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(context_prog, startup_program):
with fluid.unique_name.guard():
# image
image = fluid.layers.data(name='image',
shape=[3, 300, 300],
dtype='float32')
# backbone
backbone = MobileNet(**self.mobilenet_config)
# body_feats
body_feats = backbone(image)
# im_size
im_size = fluid.layers.data(name='im_size',
shape=[2],
dtype='int32')
# var_prefix
var_prefix = '@HUB_{}@'.format(self.name)
# names of inputs
inputs = {
'image': var_prefix + image.name,
'im_size': var_prefix + im_size.name
}
# names of outputs
if get_prediction:
locs, confs, box, box_var = fluid.layers.multi_box_head(
inputs=body_feats,
image=image,
num_classes=21,
**self.multi_box_head_config)
pred = fluid.layers.detection_output(
loc=locs,
scores=confs,
prior_box=box,
prior_box_var=box_var,
**self.output_decoder_config)
outputs = {'bbox_out': [var_prefix + pred.name]}
else:
outputs = {
'body_features':
[var_prefix + var.name for var in body_feats]
}
# add_vars_prefix
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
# inputs
inputs = {
key: context_prog.global_block().vars[value]
for key, value in inputs.items()
}
outputs = {
out_key: [
context_prog.global_block().vars[varname]
for varname in out_value
]
for out_key, out_value in outputs.items()
}
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# pretrained
if pretrained:
def _if_exist(var):
return os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
fluid.io.load_vars(exe,
self.default_pretrained_model_path,
predicate=_if_exist)
else:
exe.run(startup_program)
return inputs, outputs, context_prog
def context(self, trainable=True, pretrained=True, get_prediction=False):
"""
Distill the Head Features, so as to perform transfer learning.
Args:
trainable (bool): whether to set parameters trainable.
pretrained (bool): whether to load default pretrained model.
get_prediction (bool): whether to get prediction.
Returns:
inputs(dict): the input variables.
outputs(dict): the output variables.
context_prog (Program): the program to execute transfer learning.
"""
context_prog = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(context_prog, startup_program):
with fluid.unique_name.guard():
# image
image = fluid.layers.data(name='image',
shape=[3, 608, 608],
dtype='float32')
# backbone
backbone = DarkNet(norm_type='sync_bn',
norm_decay=0.,
depth=53)
# body_feats
body_feats = backbone(image)
# im_size
im_size = fluid.layers.data(name='im_size',
shape=[2],
dtype='int32')
# yolo_head
yolo_head = YOLOv3Head(anchor_masks=[[6, 7, 8], [3, 4, 5],
[0, 1, 2]],
anchors=[[8, 9], [10, 23], [19, 15],
[23, 33], [40, 25], [54, 50],
[101, 80], [139, 145],
[253, 224]],
norm_decay=0.,
num_classes=6,
ignore_thresh=0.7,
label_smooth=False,
nms=MultiClassNMS(
background_label=-1,
keep_top_k=100,
nms_threshold=0.45,
nms_top_k=400,
normalized=False,
score_threshold=0.005))
# head_features
head_features, body_features = yolo_head._get_outputs(
body_feats, is_train=trainable)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
# var_prefix
var_prefix = '@HUB_{}@'.format(self.name)
# name of inputs
inputs = {
'image': var_prefix + image.name,
'im_size': var_prefix + im_size.name
}
# name of outputs
if get_prediction:
bbox_out = yolo_head.get_prediction(head_features, im_size)
outputs = {'bbox_out': [var_prefix + bbox_out.name]}
else:
outputs = {
'head_features':
[var_prefix + var.name for var in head_features],
'body_features':
[var_prefix + var.name for var in body_features]
}
# add_vars_prefix
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
# inputs
inputs = {
key: context_prog.global_block().vars[value]
for key, value in inputs.items()
}
# outputs
outputs = {
key: [
context_prog.global_block().vars[varname]
for varname in value
]
for key, value in outputs.items()
}
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
# pretrained
if pretrained:
def _if_exist(var):
return os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
fluid.io.load_vars(exe,
self.default_pretrained_model_path,
predicate=_if_exist)
else:
exe.run(startup_program)
return inputs, outputs, context_prog
def context(self, trainable=True, pretrained=True):
"""context for transfer learning.
Args:
trainable (bool): Set parameters in program to be trainable.
pretrained (bool) : Whether to load pretrained model.
Returns:
inputs (dict): key is 'image', corresponding vaule is image tensor.
outputs (dict): key is :
'classification', corresponding value is the result of classification.
'feature_map', corresponding value is the result of the layer before the fully connected layer.
context_prog (fluid.Program): program for transfer learning.
"""
context_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(context_prog, startup_prog):
with fluid.unique_name.guard():
image = fluid.layers.data(
name="image", shape=[3, 224, 224], dtype="float32")
efficientnet_b0 = EfficientNetB0_small()
output, feature_map = efficientnet_b0.net(
input=image, class_dim=len(self.label_list))
name_prefix = '@HUB_{}@'.format(self.name)
inputs = {'image': name_prefix + image.name}
outputs = {
'classification': name_prefix + output.name,
'feature_map': name_prefix + feature_map.name
}
add_vars_prefix(context_prog, name_prefix)
add_vars_prefix(startup_prog, name_prefix)
global_vars = context_prog.global_block().vars
inputs = {
key: global_vars[value]
for key, value in inputs.items()
}
outputs = {
key: global_vars[value]
for key, value in outputs.items()
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# pretrained
if pretrained:
def _if_exist(var):
b = os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
return b
fluid.io.load_vars(
exe,
self.default_pretrained_model_path,
context_prog,
predicate=_if_exist)
print(inputs.keys())
fluid.io.save_inference_model(
dirname=os.path.join(
self.directory,
'efficientnetb0_small_imagenet_model'),
feeded_var_names=[name_prefix + 'image'],
target_vars=list(outputs.values()),
executor=exe,
main_program=context_prog)
else:
exe.run(startup_prog)
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
def context(self, trainable=False, max_seq_len=128, num_slots=1):
"""
Get the input ,output and program of the pretrained word2vec_skipgram
Args:
trainable(bool): Whether fine-tune the pretrained parameters of tencent_ailab_chinese_embedding_small or not.
num_slots(int): It's number of data inputted to the model, selectted as following options:
- 1(default): There's only one data to be feeded in the model, e.g. the module is used for sentence classification task.
- 2: There are two data to be feeded in the model, e.g. the module is used for text matching task (point-wise).
- 3: There are three data to be feeded in the model, e.g. the module is used for text matching task (pair-wise).
Returns:
inputs(dict): the input variables of tencent_ailab_chinese_embedding_small (words)
outputs(dict): the output variables of input words (word embeddings)
main_program(Program): the main_program of tencent_ailab_chinese_embedding_small with pretrained prameters
"""
assert num_slots >= 1 and num_slots <= 3, "num_slots must be 1, 2, or 3, but the input is %d" % num_slots
main_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
with fluid.unique_name.guard():
w_param_attrs = fluid.ParamAttr(
name="embedding_0.w_0",
initializer=fluid.initializer.TruncatedNormal(scale=0.02),
trainable=trainable)
text_1 = fluid.data(
name='text',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_1 = fluid.embedding(
input=text_1,
size=[len(self.vocab), 200],
is_sparse=True,
padding_idx=len(self.vocab) - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_1_name = emb_1.name
data_list = [text_1]
emb_name_list = [emb_1_name]
if num_slots > 1:
text_2 = fluid.data(
name='text_2',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_2 = fluid.embedding(
input=text_2,
size=[len(self.vocab), 200],
is_sparse=True,
padding_idx=len(self.vocab) - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_2_name = emb_2.name
data_list.append(text_2)
emb_name_list.append(emb_2_name)
if num_slots > 2:
text_3 = fluid.data(
name='text_3',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_3 = fluid.embedding(
input=text_3,
size=[len(self.vocab), 200],
is_sparse=True,
padding_idx=len(self.vocab) - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_3_name = emb_3.name
data_list.append(text_3)
emb_name_list.append(emb_3_name)
variable_names = filter(
lambda v: v not in ['text', 'text_2', 'text_3'],
list(main_program.global_block().vars.keys()))
prefix_name = "@HUB_{}@".format(self.name)
add_vars_prefix(
program=main_program,
prefix=prefix_name,
vars=variable_names)
for param in main_program.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# load the pretrained model
def if_exist(var):
return os.path.exists(
os.path.join(self.pretrained_model_path, var.name))
fluid.io.load_vars(
exe, self.pretrained_model_path, predicate=if_exist)
inputs = {}
outputs = {}
for index, data in enumerate(data_list):
if index == 0:
inputs['text'] = data
outputs['emb'] = main_program.global_block().vars[
prefix_name + emb_name_list[0]]
else:
inputs['text_%s' % (index + 1)] = data
outputs['emb_%s' %
(index + 1)] = main_program.global_block().vars[
prefix_name + emb_name_list[index]]
return inputs, outputs, main_program
def context(self, body_feats, fpn, rpn_head, roi_extractor, bbox_head,
bbox_assigner, image, trainable, var_prefix, phase):
"""Distill the Head Features, so as to perform transfer learning.
:param body_feats: feature map of image classification to distill feature map.
:type body_feats: list
:param fpn: Feature Pyramid Network.
:type fpn: <class 'FPN' object>
:param rpn_head: Head of Region Proposal Network.
:type rpn_head: <class 'RPNHead' object> or <class 'FPNRPNHead' object>
:param roi_extractor:
:type roi_extractor:
:param bbox_head: Head of Bounding Box.
:type bbox_head: <class 'BBoxHead' object>
:param bbox_assigner: Parameters of fluid.layers.generate_proposal_labels.
:type bbox_assigner: <class 'BBoxAssigner' object>
:param image: image tensor.
:type image: <class 'paddle.fluid.framework.Variable'>
:param trainable: whether to set parameters trainable.
:type trainable: bool
:param var_prefix: the prefix of variables in faster_rcnn
:type var_prefix: str
:param phase: Optional Choice: 'predict', 'train'
:type phase: str
"""
context_prog = image.block.program
with fluid.program_guard(context_prog):
im_info = fluid.layers.data(name='im_info',
shape=[3],
dtype='float32',
lod_level=0)
im_shape = fluid.layers.data(name='im_shape',
shape=[3],
dtype='float32',
lod_level=0)
#body_feats = backbone(image)
body_feat_names = list(body_feats.keys())
# fpn
if fpn is not None:
body_feats, spatial_scale = fpn.get_output(body_feats)
# rpn_head: RPNHead
rois = rpn_head.get_proposals(body_feats, im_info, mode=phase)
# train
if phase == 'train':
gt_bbox = fluid.layers.data(name='gt_bbox',
shape=[4],
dtype='float32',
lod_level=1)
is_crowd = fluid.layers.data(name='is_crowd',
shape=[1],
dtype='int32',
lod_level=1)
gt_class = fluid.layers.data(name='gt_class',
shape=[1],
dtype='int32',
lod_level=1)
rpn_loss = rpn_head.get_loss(im_info, gt_bbox, is_crowd)
# bbox_assigner: BBoxAssigner
outs = fluid.layers.generate_proposal_labels(
rpn_rois=rois,
gt_classes=gt_class,
is_crowd=is_crowd,
gt_boxes=gt_bbox,
im_info=im_info,
batch_size_per_im=bbox_assigner.batch_size_per_im,
fg_fraction=bbox_assigner.fg_fraction,
fg_thresh=bbox_assigner.fg_thresh,
bg_thresh_hi=bbox_assigner.bg_thresh_hi,
bg_thresh_lo=bbox_assigner.bg_thresh_lo,
bbox_reg_weights=bbox_assigner.bbox_reg_weights,
class_nums=bbox_assigner.class_nums,
use_random=bbox_assigner.use_random)
rois = outs[0]
if fpn is None:
body_feat = body_feats[body_feat_names[-1]]
# roi_extractor: RoIAlign
roi_feat = fluid.layers.roi_align(
input=body_feat,
rois=rois,
pooled_height=roi_extractor.pooled_height,
pooled_width=roi_extractor.pooled_width,
spatial_scale=roi_extractor.spatial_scale,
sampling_ratio=roi_extractor.sampling_ratio)
else:
# roi_extractor: FPNRoIAlign
roi_feat = roi_extractor(head_inputs=body_feats,
rois=rois,
spatial_scale=spatial_scale)
# head_feat
head_feat = bbox_head.head(roi_feat)
if isinstance(head_feat, OrderedDict):
head_feat = list(head_feat.values())[0]
if phase == 'train':
inputs = {
'image': var_prefix + image.name,
'im_info': var_prefix + im_info.name,
'im_shape': var_prefix + im_shape.name,
'gt_class': var_prefix + gt_class.name,
'gt_bbox': var_prefix + gt_bbox.name,
'is_crowd': var_prefix + is_crowd.name
}
outputs = {
'head_feat':
var_prefix + head_feat.name,
'rpn_cls_loss':
var_prefix + rpn_loss['rpn_cls_loss'].name,
'rpn_reg_loss':
var_prefix + rpn_loss['rpn_reg_loss'].name,
'generate_proposal_labels':
[var_prefix + var.name for var in outs]
}
elif phase == 'predict':
pred = bbox_head.get_prediction(roi_feat, rois, im_info,
im_shape)
inputs = {
'image': var_prefix + image.name,
'im_info': var_prefix + im_info.name,
'im_shape': var_prefix + im_shape.name
}
outputs = {
'head_feat': var_prefix + head_feat.name,
'rois': var_prefix + rois.name,
'bbox_out': var_prefix + pred.name
}
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
global_vars = context_prog.global_block().vars
inputs = {key: global_vars[value] for key, value in inputs.items()}
outputs = {
key: global_vars[value] if not isinstance(value, list) else
[global_vars[var] for var in value]
for key, value in outputs.items()
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
def context(self, trainable=True, pretrained=True, get_prediction=False):
"""
Distill the Head Features, so as to perform transfer learning.
Args:
trainable (bool): whether to set parameters trainable.
pretrained (bool): whether to load default pretrained model.
get_prediction (bool): whether to get prediction.
Returns:
inputs(dict): the input variables.
outputs(dict): the output variables.
context_prog (Program): the program to execute transfer learning.
"""
context_prog = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(context_prog, startup_program):
with fluid.unique_name.guard():
# image
image = fluid.layers.data(name='image',
shape=[3, 608, 608],
dtype='float32')
# backbone
backbone = ResNet(norm_type='sync_bn',
freeze_at=0,
freeze_norm=False,
norm_decay=0.,
dcn_v2_stages=[5],
depth=50,
variant='d',
feature_maps=[3, 4, 5])
# body_feats
body_feats = backbone(image)
# im_size
im_size = fluid.layers.data(name='im_size',
shape=[2],
dtype='int32')
# yolo_head
yolo_head = YOLOv3Head(num_classes=80)
# head_features
head_features, body_features = yolo_head._get_outputs(
body_feats, is_train=trainable)
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
# var_prefix
var_prefix = '@HUB_{}@'.format(self.name)
# name of inputs
inputs = {
'image': var_prefix + image.name,
'im_size': var_prefix + im_size.name
}
# name of outputs
if get_prediction:
bbox_out = yolo_head.get_prediction(head_features, im_size)
outputs = {'bbox_out': [var_prefix + bbox_out.name]}
else:
outputs = {
'head_features':
[var_prefix + var.name for var in head_features],
'body_features':
[var_prefix + var.name for var in body_features]
}
# add_vars_prefix
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
# inputs
inputs = {
key: context_prog.global_block().vars[value]
for key, value in inputs.items()
}
# outputs
outputs = {
key: [
context_prog.global_block().vars[varname]
for varname in value
]
for key, value in outputs.items()
}
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
# pretrained
if pretrained:
def _if_exist(var):
return os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
fluid.io.load_vars(exe,
self.default_pretrained_model_path,
predicate=_if_exist)
else:
exe.run(startup_program)
return inputs, outputs, context_prog
def createMultiInputs(moduleName, inputSigns, isPooled=True, max_seq_len=128):
# load ernie module
print(moduleName, inputSigns, isPooled, max_seq_len)
ernie = hub.Module(name=moduleName)
#ernie = hub.Module(name='ernie')
#'pooled_output', 'sequence_output'
if isPooled:
outSign = "pooled_output"
else:
outSign = "sequence_output"
# create three programs
programs = []
inputs = []
outputs = []
for inputSign in inputSigns:
input_dict, output_dict, program = ernie.context(
max_seq_len=max_seq_len)
programs.append(program)
inputs.append(input_dict)
outputs.append(output_dict)
outPutSign = output_dict[outSign].name
# get the names of input and output variables
feed_list = [
input_dict["input_ids"].name, input_dict["segment_ids"].name,
input_dict["position_ids"].name, input_dict["input_mask"].name
]
params = [param.name for param in program.global_block().iter_parameters()]
tmp_vars = set(
[var for var in program.global_block().vars if var not in params])
input_vars = set(varname for varname in feed_list)
input_vars = set(var.name for var in input_dict.values())
print("all input_vars", input_vars)
input_vars = set(varname for varname in feed_list)
print("use input_vars", input_vars)
output_vars = set(var.name for var in output_dict.values())
#vars = list(input_vars | output_vars)
vars = list(input_vars | output_vars | tmp_vars)
startProgram = programs[0]
feed_var_names = []
fetch_var_names = []
out_var_names = []
for i, program in enumerate(programs):
inputSign = inputSigns[i] + "_"
add_vars_prefix(program, prefix=inputSign, vars=vars)
feed_var_names += [inputSign + var for var in feed_list]
fetch_var_names += [inputSign + var for var in output_vars]
out_var_names += [inputSign + outPutSign]
if i > 0:
print("connext:", inputSign, i)
connect_program(startProgram, program, inplace=True)
return startProgram, feed_var_names, fetch_var_names, out_var_names
def context(self, trainable=False, max_seq_len=128, num_slots=1):
"""
Get the input ,output and program of the pretrained simnet_bow
Args:
trainable(bool): whether fine-tune the pretrained parameters of simnet_bow or not。
max_seq_len (int): It will limit the total sequence returned so that it has a maximum length.
num_slots(int): It's number of data inputted to the model, selectted as following options:
- 1(default): There's only one data to be feeded in the model, e.g. the module is used for sentence classification task.
- 2: There are two data to be feeded in the model, e.g. the module is used for text matching task (point-wise).
- 3: There are three data to be feeded in the model, e.g. the module is used for text matching task (pair-wise).
Returns:
inputs(dict): the input variables of simnet_bow (words)
outputs(dict): the output variables of input words (word embeddings) and sequence lenght of the first input_text
main_program(Program): the main_program of simnet_bow with pretrained prameters
"""
assert num_slots >= 1 and num_slots <= 3, "num_slots must be 1, 2, or 3, but the input is %d" % num_slots
main_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
text_1 = fluid.layers.data(name="text",
shape=[-1, max_seq_len, 1],
dtype="int64",
lod_level=0)
seq_len = fluid.layers.data(name="seq_len",
shape=[1],
dtype='int64',
lod_level=0)
seq_len_used = fluid.layers.squeeze(seq_len, axes=[1])
# Add embedding layer.
w_param_attrs = fluid.ParamAttr(
name="emb",
initializer=fluid.initializer.TruncatedNormal(scale=0.02),
trainable=trainable)
dict_dim = 500002
emb_1 = fluid.layers.embedding(input=text_1,
size=[dict_dim, 128],
is_sparse=True,
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_1_name = emb_1.name
data_list = [text_1]
emb_name_list = [emb_1_name]
if num_slots > 1:
text_2 = fluid.data(name='text_2',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_2 = fluid.embedding(input=text_2,
size=[dict_dim, 128],
is_sparse=True,
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_2_name = emb_2.name
data_list.append(text_2)
emb_name_list.append(emb_2_name)
if num_slots > 2:
text_3 = fluid.data(name='text_3',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_3 = fluid.embedding(input=text_3,
size=[dict_dim, 128],
is_sparse=True,
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_3_name = emb_3.name
data_list.append(text_3)
emb_name_list.append(emb_3_name)
variable_names = filter(
lambda v: v not in ['text', 'text_2', 'text_3', "seq_len"],
list(main_program.global_block().vars.keys()))
prefix_name = "@HUB_{}@".format(self.name)
add_vars_prefix(program=main_program,
prefix=prefix_name,
vars=variable_names)
for param in main_program.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# Load the senta_lstm pretrained model.
def if_exist(var):
return os.path.exists(
os.path.join(self.pretrained_model_path, var.name))
fluid.io.load_vars(exe,
self.pretrained_model_path,
predicate=if_exist)
inputs = {'seq_len': seq_len}
outputs = {}
for index, data in enumerate(data_list):
if index == 0:
inputs['text'] = data
outputs['emb'] = main_program.global_block().vars[
prefix_name + emb_name_list[0]]
else:
inputs['text_%s' % (index + 1)] = data
outputs['emb_%s' %
(index + 1)] = main_program.global_block().vars[
prefix_name + emb_name_list[index]]
return inputs, outputs, main_program
def context(
self,
max_seq_len=128,
trainable=True,
):
"""
get inputs, outputs and program from pre-trained module
Args:
max_seq_len (int): the max sequence length
trainable (bool): optimizing the pre-trained module params during training or not
Returns: inputs, outputs, program.
The inputs is a dict with keys named input_ids, position_ids, segment_ids, input_mask and task_ids
The outputs is a dict with two keys named pooled_output and sequence_output.
"""
assert max_seq_len <= self.MAX_SEQ_LEN and max_seq_len >= 1, "max_seq_len({}) should be in the range of [1, {}]".format(
max_seq_len, self.MAX_SEQ_LEN)
module_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(module_program, startup_program):
with fluid.unique_name.guard("@HUB_%s@" % self.name):
input_ids = fluid.layers.data(
name='input_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
position_ids = fluid.layers.data(
name='position_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
segment_ids = fluid.layers.data(
name='segment_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
input_mask = fluid.layers.data(
name='input_mask',
shape=[-1, max_seq_len, 1],
dtype='float32',
lod_level=0)
pooled_output, sequence_output = self.net(
input_ids, position_ids, segment_ids, input_mask)
inputs = {
'input_ids': input_ids,
'position_ids': position_ids,
'segment_ids': segment_ids,
'input_mask': input_mask,
}
outputs = {
"pooled_output": pooled_output,
"sequence_output": sequence_output,
0: pooled_output,
1: sequence_output
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# To be compatible with the module v1
vars = filter(lambda var: "tmp" not in var,
list(module_program.global_block().vars.keys())[4:])
paddle_helper.add_vars_prefix(
program=module_program, prefix=self.param_prefix(), vars=vars)
self.init_pretraining_params(
exe, self.params_path, main_program=module_program)
self.params_layer = {}
for param in module_program.global_block().iter_parameters():
param.trainable = trainable
match = re.match(r'.*layer_(\d+).*', param.name)
if match:
# layer num begins from 0
layer = match.group(1)
self.params_layer[param.name] = int(layer)
return inputs, outputs, module_program
def context(self,
num_classes=81,
trainable=True,
pretrained=True,
phase='train'):
"""
Distill the Head Features, so as to perform transfer learning.
Args:
num_classes (int): number of categories
trainable (bool): whether to set parameters trainable.
pretrained (bool): whether to load default pretrained model.
phase (str): optional choices are 'train' and 'predict'.
Returns:
inputs (dict): the input variables.
outputs (dict): the output variables.
context_prog (Program): the program to execute transfer learning.
"""
context_prog = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(context_prog, startup_program):
with fluid.unique_name.guard():
image = fluid.layers.data(name='image',
shape=[-1, 3, -1, -1],
dtype='float32')
# backbone
backbone = ResNet(norm_type='affine_channel',
depth=50,
feature_maps=4,
freeze_at=2)
body_feats = backbone(image)
# var_prefix
var_prefix = '@HUB_{}@'.format(self.name)
im_info = fluid.layers.data(name='im_info',
shape=[3],
dtype='float32',
lod_level=0)
im_shape = fluid.layers.data(name='im_shape',
shape=[3],
dtype='float32',
lod_level=0)
body_feat_names = list(body_feats.keys())
# rpn_head: RPNHead
rpn_head = self.rpn_head()
rois = rpn_head.get_proposals(body_feats, im_info, mode=phase)
# train
if phase == 'train':
gt_bbox = fluid.layers.data(name='gt_bbox',
shape=[4],
dtype='float32',
lod_level=1)
is_crowd = fluid.layers.data(name='is_crowd',
shape=[1],
dtype='int32',
lod_level=1)
gt_class = fluid.layers.data(name='gt_class',
shape=[1],
dtype='int32',
lod_level=1)
rpn_loss = rpn_head.get_loss(im_info, gt_bbox, is_crowd)
# bbox_assigner: BBoxAssigner
bbox_assigner = self.bbox_assigner(num_classes)
outs = fluid.layers.generate_proposal_labels(
rpn_rois=rois,
gt_classes=gt_class,
is_crowd=is_crowd,
gt_boxes=gt_bbox,
im_info=im_info,
batch_size_per_im=bbox_assigner.batch_size_per_im,
fg_fraction=bbox_assigner.fg_fraction,
fg_thresh=bbox_assigner.fg_thresh,
bg_thresh_hi=bbox_assigner.bg_thresh_hi,
bg_thresh_lo=bbox_assigner.bg_thresh_lo,
bbox_reg_weights=bbox_assigner.bbox_reg_weights,
class_nums=bbox_assigner.class_nums,
use_random=bbox_assigner.use_random)
rois = outs[0]
body_feat = body_feats[body_feat_names[-1]]
# roi_extractor: RoIAlign
roi_extractor = self.roi_extractor()
roi_feat = fluid.layers.roi_align(
input=body_feat,
rois=rois,
pooled_height=roi_extractor.pooled_height,
pooled_width=roi_extractor.pooled_width,
spatial_scale=roi_extractor.spatial_scale,
sampling_ratio=roi_extractor.sampling_ratio)
# head_feat
bbox_head = self.bbox_head(num_classes)
head_feat = bbox_head.head(roi_feat)
if isinstance(head_feat, OrderedDict):
head_feat = list(head_feat.values())[0]
if phase == 'train':
inputs = {
'image': var_prefix + image.name,
'im_info': var_prefix + im_info.name,
'im_shape': var_prefix + im_shape.name,
'gt_class': var_prefix + gt_class.name,
'gt_bbox': var_prefix + gt_bbox.name,
'is_crowd': var_prefix + is_crowd.name
}
outputs = {
'head_features':
var_prefix + head_feat.name,
'rpn_cls_loss':
var_prefix + rpn_loss['rpn_cls_loss'].name,
'rpn_reg_loss':
var_prefix + rpn_loss['rpn_reg_loss'].name,
'generate_proposal_labels':
[var_prefix + var.name for var in outs]
}
elif phase == 'predict':
pred = bbox_head.get_prediction(roi_feat, rois, im_info,
im_shape)
inputs = {
'image': var_prefix + image.name,
'im_info': var_prefix + im_info.name,
'im_shape': var_prefix + im_shape.name
}
outputs = {
'head_features': var_prefix + head_feat.name,
'rois': var_prefix + rois.name,
'bbox_out': var_prefix + pred.name
}
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(startup_program, var_prefix)
global_vars = context_prog.global_block().vars
inputs = {
key: global_vars[value]
for key, value in inputs.items()
}
outputs = {
key: global_vars[value] if not isinstance(value, list) else
[global_vars[var] for var in value]
for key, value in outputs.items()
}
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(startup_program)
if pretrained:
def _if_exist(var):
if num_classes != 81:
if 'bbox_pred' in var.name or 'cls_score' in var.name:
return False
return os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
fluid.io.load_vars(exe,
self.default_pretrained_model_path,
predicate=_if_exist)
return inputs, outputs, context_prog
def context(self,
trainable=True,
pretrained=True,
override_params=None,
phase='train'):
"""context for transfer learning.
Args:
trainable (bool): Set parameters in program to be trainable.
pretrained (bool) : Whether to load pretrained model.
Returns:
inputs (dict): key is 'image', corresponding vaule is image tensor.
outputs (dict): key is :
'classification', corresponding value is the result of classification.
'feature_map', corresponding value is the result of the layer before the fully connected layer.
context_prog (fluid.Program): program for transfer learning.
"""
if phase in ["dev", "test", "predict", "eval"]:
is_test = False
elif phase in ["train"]:
is_test = True
else:
raise ValueError(
"Phase %s is error, which must be one of train, dev, test, eval and predict."
% phase)
context_prog = fluid.Program()
startup_prog = fluid.Program()
with fluid.program_guard(context_prog, startup_prog):
with fluid.unique_name.guard():
image = fluid.layers.data(
name="image", shape=[3, 224, 224], dtype="float32")
efficientnet_b1 = EfficientNetB1(
override_params=override_params)
output, feature_map = efficientnet_b1.net(
input=image,
class_dim=len(self.label_list),
is_test=is_test)
name_prefix = '@HUB_{}@'.format(self.name)
inputs = {'image': name_prefix + image.name}
outputs = {
'classification': name_prefix + output.name,
'feature_map': name_prefix + feature_map.name
}
add_vars_prefix(context_prog, name_prefix)
add_vars_prefix(startup_prog, name_prefix)
global_vars = context_prog.global_block().vars
inputs = {
key: global_vars[value]
for key, value in inputs.items()
}
outputs = {
key: global_vars[value]
for key, value in outputs.items()
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# pretrained
if pretrained:
def _if_exist(var):
b = os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
return b
fluid.io.load_vars(
exe,
self.default_pretrained_model_path,
context_prog,
predicate=_if_exist)
else:
exe.run(startup_prog)
# trainable
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
def context(self, trainable=False, max_seq_len=128, num_data=1):
"""
Get the input ,output and program of the pretrained senta_gru
Args:
trainable(bool): whether fine-tune the pretrained parameters of senta_gru or not
num_data(int): It's number of data inputted to the model, selectted as following options:
- 1(default): There's only one data to be feeded in the model, e.g. the module is used for text classification task.
- 2: There are two data to be feeded in the model, e.g. the module is used for text matching task (point-wise).
- 3: There are three data to be feeded in the model, e.g. the module is used for text matching task (pair-wise).
Returns:
inputs(dict): the input variables of senta_gru (words)
outputs(dict): the output variables of input words (word embeddings and label probilities);
the sentence embedding and sequence length of the first input text.
main_program(Program): the main_program of Senta with pretrained prameters
"""
assert num_data >= 1 and num_data <= 3, "num_data(%d) must be 1, 2, or 3" % num_data
main_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
text_1 = fluid.layers.data(name="text_1",
shape=[-1, max_seq_len, 1],
dtype="int64",
lod_level=0)
seq_len = fluid.layers.data(name="seq_len",
shape=[1],
dtype='int64',
lod_level=0)
seq_len_used = fluid.layers.squeeze(seq_len, axes=[1])
# Add embedding layer.
w_param_attrs = fluid.ParamAttr(
name="embedding_0.w_0",
initializer=fluid.initializer.TruncatedNormal(scale=0.02),
trainable=trainable)
dict_dim = 1256607
emb_1 = fluid.layers.embedding(input=text_1,
size=[dict_dim, 128],
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_1_name = emb_1.name
data_list = [text_1]
emb_name_list = [emb_1_name]
# Add lstm layer.
pred, fc = gru_net(emb_1, seq_len_used)
pred_name = pred.name
fc_name = fc.name
if num_data > 1:
text_2 = fluid.data(name='text_2',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_2 = fluid.embedding(input=text_2,
size=[dict_dim, 128],
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_2_name = emb_2.name
data_list.append(text_2)
emb_name_list.append(emb_2_name)
if num_data > 2:
text_3 = fluid.data(name='text_3',
shape=[-1, max_seq_len],
dtype='int64',
lod_level=0)
emb_3 = fluid.embedding(input=text_3,
size=[dict_dim, 128],
padding_idx=dict_dim - 1,
dtype='float32',
param_attr=w_param_attrs)
emb_3_name = emb_3.name
data_list.append(text_3)
emb_name_list.append(emb_3_name)
variable_names = filter(
lambda v: v not in ['text_1', 'text_2', 'text_3', "seq_len"],
list(main_program.global_block().vars.keys()))
prefix_name = "@HUB_{}@".format(self.name)
add_vars_prefix(program=main_program,
prefix=prefix_name,
vars=variable_names)
for param in main_program.global_block().iter_parameters():
param.trainable = trainable
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# Load the senta_lstm pretrained model.
def if_exist(var):
print(
var.name,
os.path.exists(
os.path.join(self.pretrained_model_path, var.name)))
return os.path.exists(
os.path.join(self.pretrained_model_path, var.name))
fluid.io.load_vars(exe,
self.pretrained_model_path,
predicate=if_exist)
inputs = {'seq_len': seq_len}
outputs = {
"class_probs":
main_program.global_block().vars[prefix_name + pred_name],
"sentence_feature":
main_program.global_block().vars[prefix_name + fc_name]
}
for index, data in enumerate(data_list):
inputs['text_%s' % (index + 1)] = data
outputs['emb_%s' %
(index + 1)] = main_program.global_block().vars[
prefix_name + emb_name_list[index]]
return inputs, outputs, main_program
def context(
self,
max_seq_len=None,
trainable=True,
num_slots=1,
):
"""
get inputs, outputs and program from pre-trained module
Args:
max_seq_len (int): It will limit the total sequence returned so that it has a maximum length.
trainable (bool): Whether fine-tune the pre-trained module parameters or not.
num_slots(int): It's number of data inputted to the model, selectted as following options:
- 1(default): There's only one data to be feeded in the model, e.g. the module is used for sentence classification task.
- 2: There are two data to be feeded in the model, e.g. the module is used for text matching task (point-wise).
- 3: There are three data to be feeded in the model, e.g. the module is used for text matching task (pair-wise).
Returns: inputs, outputs, program.
The inputs is a dict with keys named input_ids, position_ids, segment_ids, input_mask and task_ids
The outputs is a dict with two keys named pooled_output and sequence_output.
"""
assert num_slots >= 1 and num_slots <= 3, "num_slots must be 1, 2, or 3, but the input is %d" % num_slots
if not max_seq_len:
max_seq_len = self.max_seq_len
assert max_seq_len <= self.MAX_SEQ_LEN and max_seq_len >= 1, "max_seq_len({}) should be in the range of [1, {}]".format(
max_seq_len, self.MAX_SEQ_LEN)
module_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(module_program, startup_program):
with fluid.unique_name.guard():
input_ids = fluid.layers.data(
name='input_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
position_ids = fluid.layers.data(
name='position_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
segment_ids = fluid.layers.data(
name='segment_ids',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
input_mask = fluid.layers.data(
name='input_mask',
shape=[-1, max_seq_len, 1],
dtype='float32',
lod_level=0)
pooled_output, sequence_output = self.net(
input_ids, position_ids, segment_ids, input_mask)
data_list = [(input_ids, position_ids, segment_ids, input_mask)]
output_name_list = [(pooled_output.name, sequence_output.name)]
if num_slots > 1:
input_ids_2 = fluid.layers.data(
name='input_ids_2',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
position_ids_2 = fluid.layers.data(
name='position_ids_2',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
segment_ids_2 = fluid.layers.data(
name='segment_ids_2',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
input_mask_2 = fluid.layers.data(
name='input_mask_2',
shape=[-1, max_seq_len, 1],
dtype='float32',
lod_level=0)
pooled_output_2, sequence_output_2 = self.net(
input_ids_2, position_ids_2, segment_ids_2,
input_mask_2)
data_list.append((input_ids_2, position_ids_2,
segment_ids_2, input_mask_2))
output_name_list.append((pooled_output_2.name,
sequence_output_2.name))
if num_slots > 2:
input_ids_3 = fluid.layers.data(
name='input_ids_3',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
position_ids_3 = fluid.layers.data(
name='position_ids_3',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
segment_ids_3 = fluid.layers.data(
name='segment_ids_3',
shape=[-1, max_seq_len, 1],
dtype='int64',
lod_level=0)
input_mask_3 = fluid.layers.data(
name='input_mask_3',
shape=[-1, max_seq_len, 1],
dtype='float32',
lod_level=0)
pooled_output_3, sequence_output_3 = self.net(
input_ids_3, position_ids_3, segment_ids_3,
input_mask_3)
data_list.append((input_ids_3, position_ids_3,
segment_ids_3, input_mask_3))
output_name_list.append((pooled_output_3.name,
sequence_output_3.name))
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# To be compatible with the module v1
vars = filter(
lambda var: var not in [
"input_ids", "position_ids", "segment_ids", "input_mask",
"input_ids_2", "position_ids_2", "segment_ids_2",
"input_mask_2", "input_ids_3", "position_ids_3",
"segment_ids_3", "input_mask_3"
], list(module_program.global_block().vars.keys()))
paddle_helper.add_vars_prefix(
program=module_program, prefix=self.param_prefix(), vars=vars)
self.init_pretraining_params(
exe, self.params_path, main_program=module_program)
self.params_layer = {}
for param in module_program.global_block().iter_parameters():
param.trainable = trainable
match = re.match(r'.*layer_(\d+).*', param.name)
if match:
# layer num begins from 0
layer = match.group(1)
self.params_layer[param.name] = int(layer)
inputs = {}
outputs = {}
for index, data in enumerate(data_list):
if index == 0:
inputs['input_ids'] = data[0]
inputs['position_ids'] = data[1]
inputs['segment_ids'] = data[2]
inputs['input_mask'] = data[3]
outputs['pooled_output'] = module_program.global_block().vars[
self.param_prefix() + output_name_list[0][0]]
outputs["sequence_output"] = module_program.global_block().vars[
self.param_prefix() + output_name_list[0][1]]
else:
inputs['input_ids_%s' % (index + 1)] = data[0]
inputs['position_ids_%s' % (index + 1)] = data[1]
inputs['segment_ids_%s' % (index + 1)] = data[2]
inputs['input_mask_%s' % (index + 1)] = data[3]
outputs['pooled_output_%s' %
(index + 1)] = module_program.global_block().vars[
self.param_prefix() + output_name_list[index][0]]
outputs["sequence_output_%s" %
(index + 1)] = module_program.global_block().vars[
self.param_prefix() + output_name_list[index][1]]
return inputs, outputs, module_program
def context(self,
num_classes=81,
trainable=True,
pretrained=True,
get_prediction=False):
"""
Distill the Head Features, so as to perform transfer learning.
Args:
num_classes (int): number of classes.
trainable (bool): whether to set parameters trainable.
pretrained (bool): whether to load default pretrained model.
get_prediction (bool): whether to get prediction.
Returns:
inputs(dict): the input variables.
outputs(dict): the output variables.
context_prog (Program): the program to execute transfer learning.
"""
context_prog = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(context_prog, startup_program):
var_prefix = '@HUB_{}@'.format(self.name)
# image
image = fluid.layers.data(
name='image',
shape=[-1, 3, -1, -1],
dtype='float32',
lod_level=0)
# im_info
im_info = fluid.layers.data(
name='im_info', shape=[3], dtype='float32', lod_level=0)
# backbone
backbone = ResNet(
norm_type='affine_channel',
freeze_at=2,
norm_decay=0.,
depth=50,
feature_maps=[3, 4, 5])
body_feats = backbone(image)
# retina_head
retina_head = RetinaHead(
anchor_generator=AnchorGenerator(
aspect_ratios=[1.0, 2.0, 0.5],
variance=[1.0, 1.0, 1.0, 1.0]),
target_assign=RetinaTargetAssign(
positive_overlap=0.5, negative_overlap=0.4),
output_decoder=RetinaOutputDecoder(
score_thresh=0.05,
nms_thresh=0.5,
pre_nms_top_n=1000,
detections_per_im=100,
nms_eta=1.0),
num_convs_per_octave=4,
num_chan=256,
max_level=7,
min_level=3,
prior_prob=0.01,
base_scale=4,
num_scales_per_octave=3)
# fpn
fpn = FPN(
max_level=7,
min_level=3,
num_chan=256,
spatial_scale=[0.03125, 0.0625, 0.125],
has_extra_convs=True)
# body_feats
body_feats, spatial_scale = fpn.get_output(body_feats)
# inputs, outputs, context_prog
inputs = {
'image': var_prefix + image.name,
'im_info': var_prefix + im_info.name
}
if get_prediction:
pred = retina_head.get_prediction(body_feats, spatial_scale,
im_info)
outputs = {'bbox_out': var_prefix + pred.name}
else:
outputs = {
'body_features':
[var_prefix + var.name for key, var in body_feats.items()]
}
# add_vars_prefix
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
global_vars = context_prog.global_block().vars
inputs = {key: global_vars[value] for key, value in inputs.items()}
outputs = {
key: global_vars[value] if not isinstance(value, list) else
[global_vars[var] for var in value]
for key, value in outputs.items()
}
place = fluid.CPUPlace()
exe = fluid.Executor(place)
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
if pretrained:
def _if_exist(var):
return os.path.exists(
os.path.join(self.default_pretrained_model_path,
var.name))
fluid.io.load_vars(
exe,
self.default_pretrained_model_path,
predicate=_if_exist)
else:
exe.run(startup_program)
return inputs, outputs, context_prog
def context(self,
body_feats,
multi_box_head,
ssd_output_decoder,
image,
trainable=True,
var_prefix='',
get_prediction=False):
"""Distill the Head Features, so as to perform transfer learning.
:param body_feats: feature mps of backbone outputs
:type body_feats: list
:param multi_box_head: SSD head of MultiBoxHead.
:type multi_box_head: <class 'MultiBoxHead' object>
:param ssd_output_decoder: SSD output decoder
:type ssd_output_decoder: <class 'SSDOutputDecoder' object>
:param image: image tensor.
:type image: <class 'paddle.fluid.framework.Variable'>
:param trainable: whether to set parameters trainable.
:type trainable: bool
:param var_prefix: the prefix of variables in ssd
:type var_prefix: str
:param get_prediction: whether to get prediction,
if True, outputs is bbox_out,
if False, outputs is body_features.
:type get_prediction: bool
"""
context_prog = image.block.program
with fluid.program_guard(context_prog):
im_size = fluid.layers.data(name='im_size',
shape=[2],
dtype='int32')
inputs = {
'image': var_prefix + image.name,
'im_size': var_prefix + im_size.name
}
if not get_prediction:
outputs = {
'body_features':
[var_prefix + var.name for var in body_feats]
}
else:
locs, confs, box, box_var = fluid.layers.multi_box_head(
inputs=body_feats,
image=image,
base_size=multi_box_head.base_size,
num_classes=multi_box_head.num_classes,
aspect_ratios=multi_box_head.aspect_ratios,
min_ratio=multi_box_head.min_ratio,
max_ratio=multi_box_head.max_ratio,
min_sizes=multi_box_head.min_sizes,
max_sizes=multi_box_head.max_sizes,
steps=multi_box_head.steps,
offset=multi_box_head.offset,
flip=multi_box_head.flip,
kernel_size=multi_box_head.kernel_size,
pad=multi_box_head.pad,
min_max_aspect_ratios_order=multi_box_head.
min_max_aspect_ratios_order)
pred = fluid.layers.detection_output(
loc=locs,
scores=confs,
prior_box=box,
prior_box_var=box_var,
nms_threshold=ssd_output_decoder.nms_threshold,
nms_top_k=ssd_output_decoder.nms_top_k,
keep_top_k=ssd_output_decoder.keep_top_k,
score_threshold=ssd_output_decoder.score_threshold,
nms_eta=ssd_output_decoder.nms_eta,
background_label=ssd_output_decoder.background_label)
outputs = {'bbox_out': var_prefix + pred.name}
add_vars_prefix(context_prog, var_prefix)
add_vars_prefix(fluid.default_startup_program(), var_prefix)
inputs = {
key: context_prog.global_block().vars[value]
for key, value in inputs.items()
}
outputs = {
key: [
context_prog.global_block().vars[varname]
for varname in value
]
for key, value in outputs.items()
}
for param in context_prog.global_block().iter_parameters():
param.trainable = trainable
return inputs, outputs, context_prog
