def _dump_processor(self):
import inspect
pymodule = inspect.getmodule(self.processor)
pycode = inspect.getsource(pymodule)
processor_path = self.helper.processor_path()
processor_md5 = utils.md5(pycode)
processor_md5 += str(time.time())
processor_name = utils.md5(processor_md5)
output_file = os.path.join(processor_path, processor_name + ".py")
utils.mkdir(processor_path)
with open(output_file, "w") as file:
file.write(pycode)
utils.from_pyobj_to_module_attr(
processor_name, self.desc.attr.map.data['processor_info'])
def from_param_to_module_attr(param, module_attr):
def paddle_obj_filter(pyobj):
return isinstance(pyobj, fluid.framework.Variable) or isinstance(
pyobj, fluid.framework.Block) or isinstance(
pyobj, fluid.framework.Program) or isinstance(
pyobj, fluid.framework.Operator)
module_attr.type = module_desc_pb2.MAP
from_pyobj_to_module_attr(param.trainable,
module_attr.map.data['trainable'])
from_pyobj_to_module_attr(param.do_model_average,
module_attr.map.data['do_model_average'])
from_pyobj_to_module_attr(param.optimize_attr,
module_attr.map.data['optimize_attr'])
from_pyobj_to_module_attr(param.regularizer,
module_attr.map.data['regularizer'],
obj_filter=paddle_obj_filter)
from_pyobj_to_module_attr(param.gradient_clip_attr,
module_attr.map.data['gradient_clip_attr'],
obj_filter=paddle_obj_filter)
def _generate_desc(self):
# save fluid Parameter
attr = self.desc.attr
attr.type = module_desc_pb2.MAP
param_attrs = attr.map.data['param_attrs']
param_attrs.type = module_desc_pb2.MAP
for param in self.program.global_block().iter_parameters():
param_attr = param_attrs.map.data[param.name]
paddle_helper.from_param_to_module_attr(param, param_attr)
# save Variable Info
var_infos = attr.map.data['var_infos']
var_infos.type = module_desc_pb2.MAP
for block in self.program.blocks:
for var in block.vars.values():
var_info = var_infos.map.data[var.name]
var_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(
var.stop_gradient, var_info.map.data['stop_gradient'])
utils.from_pyobj_to_module_attr(block.idx,
var_info.map.data['block_id'])
# save signarture info
for key, sign in self.signatures.items():
var = self.desc.sign2var[sign.name]
feed_desc = var.feed_desc
fetch_desc = var.fetch_desc
feed_names = sign.feed_names
fetch_names = sign.fetch_names
for index, input in enumerate(sign.inputs):
feed_var = feed_desc.add()
feed_var.var_name = self.get_var_name_with_prefix(input.name)
feed_var.alias = feed_names[index]
for index, output in enumerate(sign.outputs):
fetch_var = fetch_desc.add()
fetch_var.var_name = self.get_var_name_with_prefix(output.name)
fetch_var.alias = fetch_names[index]
# save default signature
utils.from_pyobj_to_module_attr(
self.default_signature.name if self.default_signature else None,
attr.map.data['default_signature'])
# save name prefix
utils.from_pyobj_to_module_attr(self.name_prefix,
self.desc.attr.map.data["name_prefix"])
# save module info
module_info = attr.map.data['module_info']
module_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(self.name, module_info.map.data['name'])
utils.from_pyobj_to_module_attr(self.version,
module_info.map.data['version'])
utils.from_pyobj_to_module_attr(self.author,
module_info.map.data['author'])
utils.from_pyobj_to_module_attr(self.author_email,
module_info.map.data['author_email'])
utils.from_pyobj_to_module_attr(self.type, module_info.map.data['type'])
utils.from_pyobj_to_module_attr(self.summary,
module_info.map.data['summary'])
# save extra info
extra_info = attr.map.data['extra_info']
extra_info.type = module_desc_pb2.MAP
for key, value in self.extra_info.items():
utils.from_pyobj_to_module_attr(value, extra_info.map.data[key])
def create_module(directory, name, author, email, module_type, summary,
version):
save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX)
with tmp_dir() as base_dir:
# package the module
with tarfile.open(save_file, "w:gz") as tar:
module_dir = os.path.join(base_dir, name)
shutil.copytree(directory, module_dir)
# record module info and serialize
desc = module_desc_pb2.ModuleDesc()
attr = desc.attr
attr.type = module_desc_pb2.MAP
module_info = attr.map.data['module_info']
module_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(name, module_info.map.data['name'])
utils.from_pyobj_to_module_attr(author,
module_info.map.data['author'])
utils.from_pyobj_to_module_attr(
email, module_info.map.data['author_email'])
utils.from_pyobj_to_module_attr(module_type,
module_info.map.data['type'])
utils.from_pyobj_to_module_attr(summary,
module_info.map.data['summary'])
utils.from_pyobj_to_module_attr(version,
module_info.map.data['version'])
module_desc_path = os.path.join(module_dir, "module_desc.pb")
with open(module_desc_path, "wb") as f:
f.write(desc.SerializeToString())
# generate check info
checker = ModuleChecker(module_dir)
checker.generate_check_info()
# add __init__
module_init = os.path.join(module_dir, "__init__.py")
with open(module_init, "a") as file:
file.write("")
_cwd = os.getcwd()
os.chdir(base_dir)
module_dir = module_dir.replace(base_dir, ".")
tar.add(module_dir, recursive=False)
files = []
for dirname, _, subfiles in os.walk(module_dir):
for file in subfiles:
# if file.startswith("."):
# continue
files.append(os.path.join(dirname, file))
total_length = len(files)
print("Create Module {}-{}".format(name, version))
for index, file in enumerate(files):
done = int(float(index) / total_length * 50)
progress("[%-50s] %.2f%%" %
('=' * done, float(index / total_length * 100)))
tar.add(file)
progress("[%-50s] %.2f%%" % ('=' * 50, 100), end=True)
print("Module package saved as {}".format(save_file))
os.chdir(_cwd)
def create_module(directory, name, author, email, module_type, summary,
version):
save_file = "{}-{}.{}".format(name, version, HUB_PACKAGE_SUFFIX)
with tmp_dir() as base_dir:
# package the module
with tarfile.open(save_file, "w:gz") as tar:
module_dir = os.path.join(base_dir, name)
shutil.copytree(directory, module_dir)
# record module info and serialize
desc = module_desc_pb2.ModuleDesc()
attr = desc.attr
attr.type = module_desc_pb2.MAP
module_info = attr.map.data['module_info']
module_info.type = module_desc_pb2.MAP
utils.from_pyobj_to_module_attr(name, module_info.map.data['name'])
utils.from_pyobj_to_module_attr(author,
module_info.map.data['author'])
utils.from_pyobj_to_module_attr(
email, module_info.map.data['author_email'])
utils.from_pyobj_to_module_attr(module_type,
module_info.map.data['type'])
utils.from_pyobj_to_module_attr(summary,
module_info.map.data['summary'])
utils.from_pyobj_to_module_attr(version,
module_info.map.data['version'])
module_desc_path = os.path.join(module_dir, "module_desc.pb")
with open(module_desc_path, "wb") as f:
f.write(desc.SerializeToString())
# generate check info
checker = ModuleChecker(module_dir)
checker.generate_check_info()
# add __init__
module_init = os.path.join(module_dir, "__init__.py")
with open(module_init, "a") as file:
file.write("")
_cwd = os.getcwd()
os.chdir(base_dir)
for dirname, _, files in os.walk(module_dir):
for file in files:
tar.add(os.path.join(dirname, file).replace(base_dir, "."))
os.chdir(_cwd)