def test_tuple_2_flexible_data(self): input = (1, 2, 3) flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) assert flexible_data.type == module_desc_pb2.LIST, "type conversion error" assert len( flexible_data.list.data) == len(input), "value convesion error" for index in range(len(input)): _check_int(input[index], flexible_data.list.data[str(index)])
def test_set_2_flexible_data(self): input = {1, 2, 3} flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) assert flexible_data.type == module_desc_pb2.SET, "type conversion error" assert len( flexible_data.set.data) == len(input), "value convesion error" for index in range(len(input)): assert flexible_data.set.data[str( index)].i in input, "value convesion error"
def test_dict_2_flexible_data(self): input = {1: 1, 2: 2, 3: 3} flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) assert flexible_data.type == module_desc_pb2.MAP, "type conversion error" assert len( flexible_data.map.data) == len(input), "value convesion error" for key, value in flexible_data.map.data.items(): realkey = get_pykey(key, flexible_data.map.keyType[key]) assert realkey in input, "key convesion error" _check_int(input[realkey], flexible_data.map.data[key])
def test_convert_compound_object(self): input = { False: 1, '2': 3, 4.0: [5, 6.0, ['7', { 8: 9 }]], 'set': {10}, 'dict': { 11: 12 } } flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert input == output, "dict convesion error"
def test_obj_2_flexible_data(self): class TestObj: def __init__(self): self.a = 1 self.b = 2.0 self.c = "str" self.d = {'a': 123} input = TestObj() flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) assert flexible_data.type == module_desc_pb2.OBJECT, "type conversion error" assert len(flexible_data.object.data) == len( input.__dict__), "value convesion error" _check_int(input.a, flexible_data.object.data['a']) _check_float(input.b, flexible_data.object.data['b']) _check_str(input.c, flexible_data.object.data['c']) _check_int(input.d['a'], flexible_data.object.data['d'].map.data['a'])
def from_param_to_flexible_data(param, flexible_data): def paddle_obj_filter(pyobj): return isinstance(pyobj, fluid.framework.Variable) or isinstance( pyobj, fluid.framework.Block) or isinstance( pyobj, fluid.framework.Program) or isinstance( pyobj, fluid.framework.Operator) flexible_data.type = module_desc_pb2.MAP from_pyobj_to_flexible_data(param.trainable, flexible_data.map.data['trainable']) from_pyobj_to_flexible_data(param.do_model_average, flexible_data.map.data['do_model_average']) from_pyobj_to_flexible_data(param.optimize_attr, flexible_data.map.data['optimize_attr']) from_pyobj_to_flexible_data(param.regularizer, flexible_data.map.data['regularizer'], obj_filter=paddle_obj_filter) from_pyobj_to_flexible_data(param.gradient_clip_attr, flexible_data.map.data['gradient_clip_attr'], obj_filter=paddle_obj_filter)
def test_bool_2_flexible_data(self): input = False flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) _check_bool(input, flexible_data)
def test_convert_list(self): input = [1, 2, 3] flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert input == output, "list convesion error"
def test_float_2_flexible_data(self): input = 2.012 flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) _check_float(input, flexible_data)
def test_string_2_flexible_data(self): input = "123" flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) _check_str(input, flexible_data)
def test_convert_float(self): input = 2.012 flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert _compare_float(input, output), "float convesion error"
def test_int_2_flexible_data(self): input = 1 flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) _check_int(input, flexible_data)
def test_convert_dict(self): input = {1: 1, 2: 2, 3: 3} flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert input == output, "dict convesion error"
def test_convert_tuple(self): input = (1, 2, 3) flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert list(input) == output, "tuple convesion error"
def create_module(sign_arr, module_dir=None, word_dict=None, exe=None): """ Create a module from main program """ assert sign_arr, "signature array should not be None" # check all variable sign_arr = to_list(sign_arr) program = sign_arr[0].get_inputs()[0].block.program feeded_var_names = set() target_vars = set() for sign in sign_arr: assert isinstance(sign, Signature), "sign_arr should be list of Signature" for input in sign.get_inputs(): feeded_var_names.add(input.name) _tmp_program = input.block.program assert program == _tmp_program, "all the variable should come from the same program" for output in sign.get_outputs(): target_vars.add(output) _tmp_program = output.block.program assert program == _tmp_program, "all the variable should come from the same program" # create module path for saving if module_dir is None: module_dir = os.path.join(".", "hub_module") mkdir(module_dir) # create module pb module_desc = module_desc_pb2.ModuleDesc() module_desc.auth_info.hub_version = __version__ module_desc.auth_info.paddle_version = paddle.__version__ logger.info("hub version is %s" % __version__) logger.info("paddle version is %s" % paddle.__version__) # save asset if word_dict is None: module_desc.contain_assets = False else: module_desc.contain_assets = True with open(ModuleConfig.assets_dict_path(module_dir), "w") as fo: for w in word_dict: w_id = word_dict[w] fo.write("{}\t{}\n".format(w, w_id)) # save fluid Parameter extra_info = module_desc.extra_info extra_info.type = module_desc_pb2.MAP param_attrs = extra_info.map.data['param_attrs'] param_attrs.type = module_desc_pb2.MAP for param in program.global_block().iter_parameters(): param_attr = param_attrs.map.data[param.name] from_param_to_flexible_data(param, param_attr) # save Variable Info var_infos = extra_info.map.data['var_infos'] var_infos.type = module_desc_pb2.MAP for block in program.blocks: for var in block.vars.values(): var_info = var_infos.map.data[var.name] var_info.type = module_desc_pb2.MAP from_pyobj_to_flexible_data(var.stop_gradient, var_info.map.data['stop_gradient']) from_pyobj_to_flexible_data(block.idx, var_info.map.data['block_id']) # save signarture info sign_map = module_desc.sign2var for sign in sign_arr: if sign.get_name() in sign_map: raise "Error! sign_arr contains repeat signatrue %s" % sign var = sign_map[sign.get_name()] feed_desc = var.feed_desc fetch_desc = var.fetch_desc feed_names = sign.get_feed_names() fetch_names = sign.get_fetch_names() for index, input in enumerate(sign.get_inputs()): feed_var = feed_desc.add() feed_var.var_name = input.name feed_var.alias = feed_names[index] for index, output in enumerate(sign.get_outputs()): fetch_var = fetch_desc.add() fetch_var.var_name = output.name fetch_var.alias = fetch_names[index] # save inference program program = program.clone() if not exe: place = fluid.CPUPlace() exe = fluid.Executor(place=place) save_model_dir = os.path.join(module_dir, "model") mkdir(save_model_dir) fluid.io.save_inference_model(save_model_dir, feeded_var_names=list(feeded_var_names), target_vars=list(target_vars), main_program=program, executor=exe) with open(os.path.join(save_model_dir, "__model__"), "rb") as file: program_desc_str = file.read() rename_program = fluid.framework.Program.parse_from_string( program_desc_str) varlist = { var: block for block in rename_program.blocks for var in block.vars if HUB_VAR_PREFIX not in var } for var, block in varlist.items(): old_name = var new_name = HUB_VAR_PREFIX + old_name block._rename_var(old_name, new_name) mkdir(save_model_dir) with open(os.path.join(save_model_dir, "__model__"), "wb") as f: f.write(rename_program.desc.serialize_to_string()) for file in os.listdir(save_model_dir): if (file == "__model__" or HUB_VAR_PREFIX in file): continue os.rename(os.path.join(save_model_dir, file), os.path.join(save_model_dir, HUB_VAR_PREFIX + file)) # Serialize module_desc pb module_pb = module_desc.SerializeToString() with open(ModuleConfig.module_desc_path(module_dir), "wb") as f: f.write(module_pb)
def test_convert_bool(self): input = False flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert input == output, "bool convesion error"
def test_convert_str(self): input = "123" flexible_data = module_desc_pb2.FlexibleData() from_pyobj_to_flexible_data(input, flexible_data) output = from_flexible_data_to_pyobj(flexible_data) assert input == output, "str convesion error"