def test_tuple_2_flexible_data(self):
input = (1, 2, 3)
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.LIST, "type conversion error"
assert len(
flexible_data.list.data) == len(input), "value convesion error"
for index in range(len(input)):
_check_int(input[index], flexible_data.list.data[str(index)])
def test_set_2_flexible_data(self):
input = {1, 2, 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.SET, "type conversion error"
assert len(
flexible_data.set.data) == len(input), "value convesion error"
for index in range(len(input)):
assert flexible_data.set.data[str(
index)].i in input, "value convesion error"
def test_dict_2_flexible_data(self):
input = {1: 1, 2: 2, 3: 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.MAP, "type conversion error"
assert len(
flexible_data.map.data) == len(input), "value convesion error"
for key, value in flexible_data.map.data.items():
realkey = get_pykey(key, flexible_data.map.keyType[key])
assert realkey in input, "key convesion error"
_check_int(input[realkey], flexible_data.map.data[key])
def test_convert_compound_object(self):
input = {
False: 1,
'2': 3,
4.0: [5, 6.0, ['7', {
8: 9
}]],
'set': {10},
'dict': {
11: 12
}
}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "dict convesion error"
def test_obj_2_flexible_data(self):
class TestObj:
def __init__(self):
self.a = 1
self.b = 2.0
self.c = "str"
self.d = {'a': 123}
input = TestObj()
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
assert flexible_data.type == module_desc_pb2.OBJECT, "type conversion error"
assert len(flexible_data.object.data) == len(
input.__dict__), "value convesion error"
_check_int(input.a, flexible_data.object.data['a'])
_check_float(input.b, flexible_data.object.data['b'])
_check_str(input.c, flexible_data.object.data['c'])
_check_int(input.d['a'], flexible_data.object.data['d'].map.data['a'])
def from_param_to_flexible_data(param, flexible_data):
def paddle_obj_filter(pyobj):
return isinstance(pyobj, fluid.framework.Variable) or isinstance(
pyobj, fluid.framework.Block) or isinstance(
pyobj, fluid.framework.Program) or isinstance(
pyobj, fluid.framework.Operator)
flexible_data.type = module_desc_pb2.MAP
from_pyobj_to_flexible_data(param.trainable,
flexible_data.map.data['trainable'])
from_pyobj_to_flexible_data(param.do_model_average,
flexible_data.map.data['do_model_average'])
from_pyobj_to_flexible_data(param.optimize_attr,
flexible_data.map.data['optimize_attr'])
from_pyobj_to_flexible_data(param.regularizer,
flexible_data.map.data['regularizer'],
obj_filter=paddle_obj_filter)
from_pyobj_to_flexible_data(param.gradient_clip_attr,
flexible_data.map.data['gradient_clip_attr'],
obj_filter=paddle_obj_filter)
def test_bool_2_flexible_data(self):
input = False
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_bool(input, flexible_data)
def test_convert_list(self):
input = [1, 2, 3]
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "list convesion error"
def test_float_2_flexible_data(self):
input = 2.012
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_float(input, flexible_data)
def test_string_2_flexible_data(self):
input = "123"
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_str(input, flexible_data)
def test_convert_float(self):
input = 2.012
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert _compare_float(input, output), "float convesion error"
def test_int_2_flexible_data(self):
input = 1
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
_check_int(input, flexible_data)
def test_convert_dict(self):
input = {1: 1, 2: 2, 3: 3}
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "dict convesion error"
def test_convert_tuple(self):
input = (1, 2, 3)
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert list(input) == output, "tuple convesion error"
def create_module(sign_arr, module_dir=None, word_dict=None, exe=None):
""" Create a module from main program
"""
assert sign_arr, "signature array should not be None"
# check all variable
sign_arr = to_list(sign_arr)
program = sign_arr[0].get_inputs()[0].block.program
feeded_var_names = set()
target_vars = set()
for sign in sign_arr:
assert isinstance(sign,
Signature), "sign_arr should be list of Signature"
for input in sign.get_inputs():
feeded_var_names.add(input.name)
_tmp_program = input.block.program
assert program == _tmp_program, "all the variable should come from the same program"
for output in sign.get_outputs():
target_vars.add(output)
_tmp_program = output.block.program
assert program == _tmp_program, "all the variable should come from the same program"
# create module path for saving
if module_dir is None:
module_dir = os.path.join(".", "hub_module")
mkdir(module_dir)
# create module pb
module_desc = module_desc_pb2.ModuleDesc()
module_desc.auth_info.hub_version = __version__
module_desc.auth_info.paddle_version = paddle.__version__
logger.info("hub version is %s" % __version__)
logger.info("paddle version is %s" % paddle.__version__)
# save asset
if word_dict is None:
module_desc.contain_assets = False
else:
module_desc.contain_assets = True
with open(ModuleConfig.assets_dict_path(module_dir), "w") as fo:
for w in word_dict:
w_id = word_dict[w]
fo.write("{}\t{}\n".format(w, w_id))
# save fluid Parameter
extra_info = module_desc.extra_info
extra_info.type = module_desc_pb2.MAP
param_attrs = extra_info.map.data['param_attrs']
param_attrs.type = module_desc_pb2.MAP
for param in program.global_block().iter_parameters():
param_attr = param_attrs.map.data[param.name]
from_param_to_flexible_data(param, param_attr)
# save Variable Info
var_infos = extra_info.map.data['var_infos']
var_infos.type = module_desc_pb2.MAP
for block in program.blocks:
for var in block.vars.values():
var_info = var_infos.map.data[var.name]
var_info.type = module_desc_pb2.MAP
from_pyobj_to_flexible_data(var.stop_gradient,
var_info.map.data['stop_gradient'])
from_pyobj_to_flexible_data(block.idx,
var_info.map.data['block_id'])
# save signarture info
sign_map = module_desc.sign2var
for sign in sign_arr:
if sign.get_name() in sign_map:
raise "Error! sign_arr contains repeat signatrue %s" % sign
var = sign_map[sign.get_name()]
feed_desc = var.feed_desc
fetch_desc = var.fetch_desc
feed_names = sign.get_feed_names()
fetch_names = sign.get_fetch_names()
for index, input in enumerate(sign.get_inputs()):
feed_var = feed_desc.add()
feed_var.var_name = input.name
feed_var.alias = feed_names[index]
for index, output in enumerate(sign.get_outputs()):
fetch_var = fetch_desc.add()
fetch_var.var_name = output.name
fetch_var.alias = fetch_names[index]
# save inference program
program = program.clone()
if not exe:
place = fluid.CPUPlace()
exe = fluid.Executor(place=place)
save_model_dir = os.path.join(module_dir, "model")
mkdir(save_model_dir)
fluid.io.save_inference_model(save_model_dir,
feeded_var_names=list(feeded_var_names),
target_vars=list(target_vars),
main_program=program,
executor=exe)
with open(os.path.join(save_model_dir, "__model__"), "rb") as file:
program_desc_str = file.read()
rename_program = fluid.framework.Program.parse_from_string(
program_desc_str)
varlist = {
var: block
for block in rename_program.blocks for var in block.vars
if HUB_VAR_PREFIX not in var
}
for var, block in varlist.items():
old_name = var
new_name = HUB_VAR_PREFIX + old_name
block._rename_var(old_name, new_name)
mkdir(save_model_dir)
with open(os.path.join(save_model_dir, "__model__"), "wb") as f:
f.write(rename_program.desc.serialize_to_string())
for file in os.listdir(save_model_dir):
if (file == "__model__" or HUB_VAR_PREFIX in file):
continue
os.rename(os.path.join(save_model_dir, file),
os.path.join(save_model_dir, HUB_VAR_PREFIX + file))
# Serialize module_desc pb
module_pb = module_desc.SerializeToString()
with open(ModuleConfig.module_desc_path(module_dir), "wb") as f:
f.write(module_pb)
def test_convert_bool(self):
input = False
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "bool convesion error"
def test_convert_str(self):
input = "123"
flexible_data = module_desc_pb2.FlexibleData()
from_pyobj_to_flexible_data(input, flexible_data)
output = from_flexible_data_to_pyobj(flexible_data)
assert input == output, "str convesion error"