class GenerateDictTest(unittest.TestCase):
def setUp(self):
rospack = rospkg.RosPack()
node_path = rospack.get_path('package_generator_templates')
self.dir_template_spec = node_path + "/templates/cpp_node_update/config/"
self.spec = TemplateSpec()
# creating a temporary repo for trials
self.dir_name = "/tmp/test_template_spec"
if not os.path.exists(self.dir_name):
print "Creating the repo {}".format(self.dir_name)
os.makedirs(self.dir_name)
def test_undef_files(self):
self.assertFalse(self.spec.load_yaml_desc("unknown_file.yaml"))
self.assertFalse(self.spec.load_functions("unknown_file.py"))
def test_yaml_error(self):
bad_file = """bad_content: ["name", "author","""
filename = self.dir_name + "/bad_dico.yaml"
with open(filename, 'w') as open_file:
open_file.write(bad_file)
self.assertFalse(self.spec.load_yaml_desc(filename))
def test_correct_folder(self):
self.assertTrue(self.spec.load_spec(self.dir_template_spec))
def setUp(self):
"""
Common initialization for all tester
"""
file_content = (
'<?xml version="1.0" encoding="UTF-8"?>' '\n'
'<package name="great_package" author="anthony" author_email="*****@*****.**"' '\n'
' description="The extended package" license="TODO" copyright="TRI">' '\n'
' <component name="node_extended" frequency="100.0">' '\n'
' <publisher name="pub" type="std_msgs::Bool" description="a description"/>' '\n'
' <publisher name="pub_second" type="std_msgs::String" description="a description"/>' '\n'
' <subscriber name="sub_in" type="std_msgs::String" description="a description" />' '\n'
' <serviceClient name="service_client" type="std_srvs::Trigger" description="a description"/>' '\n'
' <serviceServer name="service_server" type="std_srvs::SetBool" description="a description"/>' '\n'
' <parameter name="param_one" type="std::string" value="Empty" description="a description"/>' '\n'
' <parameter name="param_two" type="bool" value="1" description="a description"/>' '\n'
' <actionServer name="action_server" type="bride_tutorials::TriggerPublish"' '\n'
' description="a description"/>' '\n'
' <actionClient name="action_client" type="bride_tutorials::TriggerPublish"' '\n'
' description="a description"/>' '\n'
' </component>' '\n'
'<depend>std_msgs</depend>' '\n'
'<depend>std_srvs</depend>' '\n'
'<depend>bride_tutorials</depend>' '\n'
'<depend>actionlib</depend>' '\n'
'<depend>actionlib_msgs</depend>' '\n'
'<depend>roscpp</depend>' '\n'
'</package>' '\n')
# print "File content: \n{}".format(file_content)
self.dir_name = "/tmp/test_package_generator"
if not os.path.exists(self.dir_name):
print "Creating the repo {}".format(self.dir_name)
os.makedirs(self.dir_name)
file_xml = self.dir_name + "/node_spec.ros_package"
with open(file_xml, 'w') as open_file:
open_file.write(file_content)
rospack = rospkg.RosPack()
self.template_path_ = rospack.get_path('package_generator_templates')
self.template_path_ += "/templates/cpp_node_update/"
self.spec = TemplateSpec()
self.assertTrue(self.spec.load_spec(self.template_path_ + "config"))
self.xml_parser = PackageXMLParser()
self.assertTrue(self.xml_parser.set_template_spec(self.spec))
self.assertTrue(self.xml_parser.load(file_xml))
self.generator_ = JinjaGenerator()
self.assertTrue(self.generator_.configure(self.xml_parser, self.spec))
def setUp(self):
rospack = rospkg.RosPack()
node_path = rospack.get_path('package_generator_templates')
self.dir_template_spec = node_path + "/templates/cpp_node_update/config/"
self.spec = TemplateSpec()
# creating a temporary repo for trials
self.dir_name = "/tmp/test_template_spec"
if not os.path.exists(self.dir_name):
print "Creating the repo {}".format(self.dir_name)
os.makedirs(self.dir_name)
def main():
"""main only defined for debugging purposes
Returns:
None: nothing
"""
gen = CodeGenerator()
xml_parser = PackageXMLParser()
import rospkg
rospack = rospkg.RosPack()
node_path = rospack.get_path('package_generator_templates')
dir_template_spec = node_path + "/templates/cpp_node_update/config"
spec = TemplateSpec()
if not spec.load_spec(dir_template_spec):
print "Could not load the template spec"
print "Bye"
return
if not xml_parser.set_template_spec(spec):
print "template spec not compatible with parser requirements"
print "Bye"
return
node_path = rospack.get_path('package_generator')
filename = node_path + '/tests/data/demo.ros_package'
if not xml_parser.load(filename):
print "Error while parsing the xml file {}".format(filename)
print "Bye"
return
xml_parser.set_active_node(0)
gen.configure(xml_parser, spec)
node_path = rospack.get_path('package_generator_templates')
filename = node_path + '/templates/cpp_node_update/template/README.md'
gen.reset_output_file()
if gen.process_file(filename):
output_file = "README.md"
gen.write_output_file(output_file)
print "Output written in file {}".format(output_file)
else:
print "Debug!"
def test_package_multi_nodes(self):
"""test generation of a package file involving multiple nodes spec.
"""
spec = TemplateSpec()
self.assertTrue(spec.load_spec(self.path_template_ + "config"))
xml_parser = PackageXMLParser()
self.assertTrue(xml_parser.set_template_spec(spec))
self.assertTrue(xml_parser.load(self.multi_node_package_spec_))
file_generator = CodeGenerator()
self.assertTrue(file_generator.configure(xml_parser, spec))
readme_file = self.path_template_ + "template/package.xml"
output_file = self.dir_name + "/test_package.xml"
self.assertTrue(file_generator.generate_disk_file(readme_file, output_file))
def test_parsing(self):
"""
check if a parsing goes well
Mainly implemented as an example of use
"""
node_path = self.rospack.get_path('package_generator_templates')
# the current example only contains the dictionary
dir_template_spec = node_path + "/templates/cpp_node_update/config/"
spec = TemplateSpec()
self.assertTrue(spec.load_spec(dir_template_spec))
self.assertTrue(self.package_parser.set_template_spec(spec))
node_path = self.rospack.get_path('package_generator')
file_xml = node_path + '/tests/data/demo.ros_package'
self.assertTrue(self.package_parser.load(file_xml))
def template_sanity_check(self, template):
"""Perform the package sanity check
Returns:
Bool: True on success
"""
# Locate template location
try:
[all_template_path, template_names] = self.get_template_info()
except rospkg.common.ResourceNotFound as error:
msg = "Package package_generator_templates not found in rospack"
self.log_error(msg)
self.log_error(error)
return False
except OSError as error:
msg = "No template found in package_generator_templates"
self.log_error(msg)
self.log_error(error)
return False
is_template_found = False
template_path = None
if template in template_names:
is_template_found = True
template_path = all_template_path + "/" + template
else:
self.log("Could not find template {} in {}".format(
template, all_template_path))
# check if the template provided is a relative path, and not a package in the repo
if os.path.isabs(template):
self.log(
"Loading template from absolute path {}".format(template))
is_template_found = True
template_path = template
else:
# relative path ?
template_path = os.getcwd() + "/" + template
if os.path.isdir(template_path):
self.log(
"Loading template from path {}".format(template_path))
is_template_found = True
if not is_template_found:
msg = "Template requested: {} unknown".format(template)
self.log_error(msg)
msg = "Available templates: {}".format(template_names)
self.log_error(msg)
return False
# confirm this is a template...
if not self.check_template_structure(template_path):
msg = "Please revise template structure"
self.log_error(msg)
return False
# TODO list number of files in template
# Extracting all components from the template
file_list = list()
dir_list = list()
path_root_template = template_path + "/template"
for (root, dirs, files) in os.walk(path_root_template):
# print "check {}: dir {}, files: {}".format(root, dirs, files)
if os.path.samefile(root, path_root_template):
for item in files:
file_list.append(item)
for item in dirs:
dir_list.append(item)
else:
rel_path = os.path.relpath(root, path_root_template)
for item in files:
file_list.append(rel_path + "/" + item)
for item in dirs:
dir_list.append(rel_path + "/" + item)
# print ("Dirs: ")
# print("\n".join(dir_list))
# print("Files: ")
# print("\n".join(file_list))
# setting the needed component.
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
self.jinja_generator_ = JinjaGenerator()
dir_template_spec = template_path + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.set_empty_spec():
msg_err = "Failed generating empty spec"
self.log_error(msg_err)
return False
if not self.file_generator_.configure(self.xml_parser_, self.spec_):
return False
if not self.jinja_generator_.configure(self.xml_parser_, self.spec_):
return False
is_ok = True
for item in file_list:
self.log("Checking file: {}".format(item))
item_abs = path_root_template + "/" + item
is_ok = self.check_template_file(item_abs)
if not is_ok:
break
if is_ok:
self.log("No error detected")
else:
self.log_error("Revise the template")
return is_ok
class PackageGenerator(EnhancedObject):
"""Handle the genration of a whole package
Attributes:
file_generator_ (CodeGenerator): custom generator
jinja_generator_ (JinjaGenerator): generator based on jinja
package_path_ (str): base location of the package to create
path_pkg_backup_ (str): if the package already existed, location of the package backup
spec_ (TemplateSpec): configuration of the template model
template_path_ (str): path to the template to use
xml_parser_ (PackageXMLParser): parser of the package description
"""
def __init__(self, name="PackageGenerator"):
"""Intialisation of the object
Args:
name (str, optional): Name of the component, for printing aspect
"""
# call super class constructor
super(PackageGenerator, self).__init__(name)
# path to the template to use
self.template_path_ = None
# base location of the package to create
self.package_path_ = None
# parser of the package description
self.xml_parser_ = None
# config parameter provide with the template
self.spec_ = None
# generic file generator
self.file_generator_ = None
# jinja-based generator
self.jinja_generator_ = None
# if the package already existed, location of the package backup
self.path_pkg_backup_ = None
def check_template_structure(self, template_path):
"""Check a provided path refers to a valid template structure
Args:
template_path (str): path to the package template
Returns:
Bool: True if basic sanity checks are successful
"""
if not os.path.exists(template_path):
msg = "Template path ({}) is incorrect ".format(template_path)
self.log_error(msg)
return False
if not os.path.isdir(template_path):
msg = "Template path ({}) is not a directory ".format(
template_path)
self.log_error(msg)
return False
# check if minimum information is present.
details = """A template should contain:
* config/dictionary.yaml : the dictionary to be used
* config/functions.py [optional] : additional functions used in the generation
* config/generator.py [optional] : generator list (custom, jinja) default is custom
* template/* set of elements to be generated
Revise the template, and compare to examples
"""
is_ok = True
# check for directories
required_folders = ["config", "template"]
for item in required_folders:
req_folder = template_path + "/" + item
if not os.path.isdir(req_folder):
msg_err = "Error \n Expecting to have folder " + item
msg_err += " in " + template_path
self.log_error(msg_err)
is_ok = False
# check for files
required_files = ["config/dictionary.yaml"]
for item in required_files:
req_file = template_path + "/" + item
if not os.path.isfile(req_file):
msg_err = "Error.\n Expecting to have file " + item
msg_err += " in " + template_path
self.log_error(msg_err)
is_ok = False
if not is_ok:
self.log_error("\n{}".format(details))
return False
return True
def get_template_info(self):
"""Get information about the available package templates
Returns:
list: tuple with [absolute package path, list of package names]
"""
rospack = rospkg.RosPack()
path_template = rospack.get_path('package_generator_templates')
path_template += "/templates/"
template_names = os.listdir(path_template)
return [path_template, template_names]
def generate_package(self, package_desc, output_path):
"""launches the package generation
Args:
package_desc (str): xml file containing the package description
output_path (str): directory into which the package is created
Returns:
Bool: True if the operation succeeded
"""
if not os.path.exists(output_path):
msg_err = "Incorrect desired package path ({})".format(output_path)
self.log_error(msg_err)
return False
if not os.path.isdir(output_path):
msg_err = "Desired package path ({}) not a directory ".format(
output_path)
self.log_error(msg_err)
return False
# Initialising needed components
# todo bring it to the constructor?
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
self.jinja_generator_ = JinjaGenerator()
# Start finding the template
template = self.xml_parser_.get_template(package_desc)
if template is None:
return False
# Locate template location
try:
[all_template_path, template_names] = self.get_template_info()
except rospkg.common.ResourceNotFound as error:
msg = "Package package_generator_templates not found in rospack"
self.log_error(msg)
self.log_error(error)
return False
except OSError as error:
msg = "No template dounf in package_generator_templates"
self.log_error(msg)
self.log_error(error)
return False
if template not in template_names:
msg = "Template requested: {} unknown".format(template)
self.log_error(msg)
msg = "Available templates: {}".format(template_names)
self.log_error(msg)
return False
template_path = all_template_path + "/" + template
# confirm this is a template...
if not self.check_template_structure(template_path):
msg = "Please revise template structure"
self.log_error(msg)
return False
# template localized, ready to work!
self.template_path_ = template_path
self.path_pkg_backup_ = None
dir_template_spec = self.template_path_ + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.load(package_desc):
msg_err = "Prb while parsing xml file {}".format(package_desc)
self.log_error(msg_err)
return False
# todo why only the custom generator is configured?
if not self.file_generator_.configure(self.xml_parser_, self.spec_):
return False
package_name = self.xml_parser_.get_package_spec()["name"]
self.package_path_ = output_path + "/" + package_name
if os.path.exists(self.package_path_):
self.log_warn("Package {} already exists".format(
self.package_path_))
# moving preexisting code.
# generating dir name using date
now = datetime.datetime.now()
str_date = now.strftime("%Y_%m_%d_%H_%M_%S")
self.path_pkg_backup_ = "/tmp/{}_{}".format(
os.path.basename(self.package_path_), str_date)
self.log_warn("Original package temporally stored in {}".format(
self.path_pkg_backup_))
# TODO check if the move succeeded
shutil.move(self.package_path_, self.path_pkg_backup_)
else:
self.log("Package to be created in {}".format(self.package_path_))
os.makedirs(self.package_path_)
nb_comp = self.xml_parser_.get_number_comps()
self.log("Number of components defined: {}".format(nb_comp))
if not self.generate_content():
return False
# we store the model into the directory model
path = self.package_path_ + "/model"
if not os.path.exists(path):
os.makedirs(path)
path += "/" + package_name + ".ros_package"
if self.xml_parser_.is_dependency_complete_:
try:
if os.path.abspath(package_desc) == os.path.abspath(path):
# self.log_warn("Using generated model...")
stored_desc = self.path_pkg_backup_ + "/model/" + package_name + ".ros_package"
# self.log("check {} is absolute: {}".format(package_desc, os.path.abspath(package_desc)))
shutil.copyfile(stored_desc, path)
else:
shutil.copyfile(package_desc, path)
self.log("Package model saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
else:
# some dependencies were automatically added
# model needs to be rewritten
try:
self.xml_parser_.write_xml(path)
self.log("Package model updated & saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
is_ok = self.handle_maintained_files()
return is_ok
def generate_one_file(self, template_file, result_file, force_write):
"""Generate a template file, depending on the generators to be used
Args:
template_file (str): template filename
result_file (str): filename to store the result (unless is None)
force_write (str): force the writting of empty files (if not, files is not written)
Returns:
Bool: True on success
"""
generator = dict()
generator["custom"] = self.file_generator_
generator["jinja"] = self.jinja_generator_
if len(self.spec_.generators_) == 1:
return generator[self.spec_.generators_[0]].generate_disk_file(
template_file, result_file, force_write)
# two generators are to be used
gen_one = generator[self.spec_.generators_[0]]
gen_two = generator[self.spec_.generators_[1]]
is_ok = gen_one.generate_disk_file(template_file)
if not is_ok:
return False
return gen_two.generate_open_file(gen_one.rendered_, result_file,
force_write)
def check_template_file(self, template_file):
"""Generate a template file, depending on the generators to be used
Args:
template_file (str): template filename
result_file (str): filename to store the result (unless is None)
force_write (str): force the writting of empty files (if not, files is not written)
Returns:
Bool: True on success
"""
generator = dict()
generator["custom"] = self.file_generator_
generator["jinja"] = self.jinja_generator_
if len(self.spec_.generators_) == 1:
# self.log("Check with Generator {}".format(self.spec_.generators_[0]))
return generator[self.spec_.generators_[0]].check_template_file(
template_file)
# two generators are to be used
gen_one = generator[self.spec_.generators_[0]]
gen_two = generator[self.spec_.generators_[1]]
# self.log("Check with Generator {}".format(self.spec_.generators_[0]))
is_ok = gen_one.check_template_file(template_file)
if not is_ok:
return False
# self.log("Check with Generator {}".format(self.spec_.generators_[1]))
if self.spec_.generators_[1] == "jinja":
return gen_two.check_template_file(gen_one.rendered_,
is_filename=False)
return gen_two.check_template_file(template_file)
def write_generated_file(self, result_file):
"""Write a generated file
Args:
result_file (str): filename to store the file.
Returns:
Bool: True on success
"""
generator = dict()
generator["custom"] = self.file_generator_
generator["jinja"] = self.jinja_generator_
return generator[self.spec_.generators_[-1]].write_rendered_file(
result_file)
def get_generated_file(self):
"""Get the generated files
Returns:
list: list of of each line of the generated file
"""
generator = dict()
generator["custom"] = self.file_generator_
generator["jinja"] = self.jinja_generator_
return generator[self.spec_.generators_[-1]].rendered_
def set_generated_file(self, l_file):
"""set the generated file
Args:
l_file (list): list of of each line of the generated file
"""
generator = dict()
generator["custom"] = self.file_generator_
generator["jinja"] = self.jinja_generator_
generator[self.spec_.generators_[-1]].rendered_ = l_file
def handle_maintained_files(self):
"""Restore file Developer requests to maintain
Assuming these patterns are defined in file .gen_maintain
Returns:
Bool: True on sucess
"""
# check for files to be maintained
if self.path_pkg_backup_ is None:
# package just created, no maintained file
return True
filename_rel = ".gen_maintain"
filename_abs = self.path_pkg_backup_ + "/" + filename_rel
if os.path.exists(filename_abs) and os.path.isfile(filename_abs):
self.log("Checking content to maintain after update")
else:
self.log("no maintained file defined in previous package version")
return True
with open(filename_abs) as open_file:
for line in open_file:
line = line.rstrip('\n')
if not line:
continue
path_abs = self.path_pkg_backup_ + "/" + line
if not os.path.exists(path_abs):
msg = "Content {} not found. Revise {} content"
self.log_error(msg.format(line, filename_abs))
continue
new_path = self.package_path_ + "/" + line
if os.path.isfile(path_abs):
try:
self.log("Restoring file {}".format(line))
# check if directories needs to be created
dirname = os.path.dirname(line)
# self.log("dirname is : {}".format(dirname))
if dirname:
path_abs_dir = self.package_path_ + "/" + dirname
if not os.path.isdir(path_abs_dir):
os.makedirs(path_abs_dir)
shutil.copyfile(path_abs, new_path)
except IOError as error:
msg = "Could not restore a file: {}"
self.log_error(msg.format(error))
continue
if os.path.isdir(path_abs):
try:
self.log("Restoring folder {}".format(line))
shutil.copytree(path_abs, new_path)
except IOError as error:
msg = "Could not restore folder: {}"
self.log_error(msg.format(error))
continue
self.log_error("Unkown statement {}".format(line))
# restoring the maintained content file
try:
self.log("Restoring file {}".format(filename_rel))
new_path = self.package_path_ + "/" + filename_rel
shutil.copyfile(filename_abs, new_path)
except IOError as error:
msg = "Could not restore file: {}"
self.log_error(msg.format(error))
return True
def handle_status_and_advise(self, input_file, output_file, gen_flag):
"""Depending on the file generation process outcome,
Adjust file status and inform user
Args:
input_file (str): path of the template file used
output_file (str): path of the generated file
gen_flag (Bool): Success of the generation process
Returns:
Bool: True on success of the file generation
"""
if not gen_flag:
msg = "Prb while generating file {}".format(output_file)
self.log_error(msg)
return False
# so the file generation went well
if self.file_generator_.get_len_gen_file() == 0:
# Only file __init__.py is kept empty
if os.path.basename(output_file) != '__init__.py':
msg = "File {} not written since empty".format(output_file)
self.log_warn(msg)
self.log_warn("Check: {}".format(
os.path.basename(output_file)))
return True
# file has content
file_status = os.stat(input_file)
os.chmod(output_file, file_status.st_mode)
# self.log("File {} handled".format(input_file))
self.log("File handled")
self.log("*********************************")
return True
def generate_content(self):
"""Generation and storage of all content
Returns:
Bool -- True on success
"""
# Extracting all components from the template
file_list = list()
dir_list = list()
path_root_template = self.template_path_ + "/template"
for (root, dirs, files) in os.walk(path_root_template):
# print "check {}: dir {}, files: {}".format(root, dirs, files)
if os.path.samefile(root, path_root_template):
for item in files:
file_list.append(item)
for item in dirs:
dir_list.append(item)
else:
rel_path = os.path.relpath(root, path_root_template)
for item in files:
file_list.append(rel_path + "/" + item)
for item in dirs:
dir_list.append(rel_path + "/" + item)
# Looking at final directory and filenames
package_name = self.xml_parser_.get_package_spec()["name"]
nb_comp = self.xml_parser_.get_number_comps()
comps_name = [
self.xml_parser_.data_comp_[id_comp]["attributes"]["name"]
for id_comp in range(nb_comp)
]
self.log("Generating all folders")
tmp = list()
for item in dir_list:
item = item.replace('package_name', package_name)
if 'component' in item:
for one_name in comps_name:
tmp.append(item.replace('component', one_name))
else:
tmp.append(item)
dir_list = tmp
for item in dir_list:
path_folder = self.package_path_ + "/" + item
if not os.path.exists(path_folder):
os.makedirs(path_folder)
generation_list = list()
# File preparation: storing [template filename, new filename, comp id]
for item in file_list:
new_item = item.replace('package_name', package_name)
if 'component' in item:
for num, one_name in enumerate(comps_name):
generation_list.append(
[item,
new_item.replace('component', one_name), num])
else:
# todo if no component active I should not set one
generation_list.append([item, new_item, 0])
is_ok = True
# self.log("\nFiles generation plan: ")
for item in generation_list:
[template_file, result_file, comp_id] = item
self.log("{} --> {}".format(template_file, result_file))
if not self.xml_parser_.set_active_comp(comp_id):
return False
# reconfiguring the generator to adjust to the new active component
# todo configure already called in generate_package function. Check why
if not self.file_generator_.configure(self.xml_parser_,
self.spec_):
return False
if not self.jinja_generator_.configure(self.xml_parser_,
self.spec_):
return False
# Normally an empty file should not be written
# The exception is currently only for the special python file __init__.py
is_write_forced = (os.path.basename(result_file) == '__init__.py')
result_file = self.package_path_ + "/" + result_file
template_file = self.template_path_ + '/template/' + template_file
if self.path_pkg_backup_ is None:
self.log("Generating file {}".format(result_file))
is_ok = self.generate_one_file(template_file, result_file,
is_write_forced)
if self.handle_status_and_advise(template_file, result_file,
is_ok):
continue
else:
return False
# A previous version of the package exists
# Checking if an update is necessary
rel_path = os.path.relpath(result_file, package_name)
previous_filename = os.path.join(self.path_pkg_backup_, rel_path)
# Check 1: does this file exist?
if not os.path.isfile(previous_filename):
msg = "File {} not previously existing. Just write it"
self.log_warn(msg.format(rel_path))
is_ok = self.generate_one_file(template_file, result_file,
is_write_forced)
if self.handle_status_and_advise(template_file, result_file,
is_ok):
continue
else:
return False
# File already existing. Processing previous version
is_update_needed = False
file_analyzor = GeneratedFileAnalysis()
is_ok = file_analyzor.extract_protected_region(previous_filename)
if is_ok:
# Check if Developer inserted any contribution
if file_analyzor.extracted_areas_:
# contribution found, merge needed
is_update_needed = True
else:
self.log("No Developer contribution found")
else:
msg = "prb while extracting protected area in {}"
self.log_error(msg.format(previous_filename))
self.log_error("Previous file to be manually merged, sorry")
# now we know if an update is needed
if is_ok and is_update_needed:
# self.log("Updating file {} in {}".format(rel_path, output_item))
self.log("Updating file {}".format(rel_path))
is_ok = self.generate_one_file(template_file, None, None)
if not is_ok:
return False
# todo handle this in case jinja is involved.
l_gen = self.get_generated_file()
if not l_gen:
msg = "New generated file empty. No code maintained from previous version"
self.log_warn(msg)
# we write it if forced
if is_write_forced:
is_ok = self.write_generated_file(result_file)
else:
self.log("Merging with previous version")
l_gen = file_analyzor.update_file(l_gen)
self.set_generated_file(l_gen)
is_ok = self.write_generated_file(result_file)
if self.handle_status_and_advise(template_file, result_file,
is_ok):
continue
else:
return False
# Although the file existed before, we do not have to maintain it
is_ok = self.generate_one_file(template_file, result_file,
is_write_forced)
if self.handle_status_and_advise(template_file, result_file,
is_ok):
continue
else:
return False
return True
def template_sanity_check(self, template):
"""Perform the package sanity check
Returns:
Bool: True on success
"""
# Locate template location
try:
[all_template_path, template_names] = self.get_template_info()
except rospkg.common.ResourceNotFound as error:
msg = "Package package_generator_templates not found in rospack"
self.log_error(msg)
self.log_error(error)
return False
except OSError as error:
msg = "No template found in package_generator_templates"
self.log_error(msg)
self.log_error(error)
return False
is_template_found = False
template_path = None
if template in template_names:
is_template_found = True
template_path = all_template_path + "/" + template
else:
self.log("Could not find template {} in {}".format(
template, all_template_path))
# check if the template provided is a relative path, and not a package in the repo
if os.path.isabs(template):
self.log(
"Loading template from absolute path {}".format(template))
is_template_found = True
template_path = template
else:
# relative path ?
template_path = os.getcwd() + "/" + template
if os.path.isdir(template_path):
self.log(
"Loading template from path {}".format(template_path))
is_template_found = True
if not is_template_found:
msg = "Template requested: {} unknown".format(template)
self.log_error(msg)
msg = "Available templates: {}".format(template_names)
self.log_error(msg)
return False
# confirm this is a template...
if not self.check_template_structure(template_path):
msg = "Please revise template structure"
self.log_error(msg)
return False
# TODO list number of files in template
# Extracting all components from the template
file_list = list()
dir_list = list()
path_root_template = template_path + "/template"
for (root, dirs, files) in os.walk(path_root_template):
# print "check {}: dir {}, files: {}".format(root, dirs, files)
if os.path.samefile(root, path_root_template):
for item in files:
file_list.append(item)
for item in dirs:
dir_list.append(item)
else:
rel_path = os.path.relpath(root, path_root_template)
for item in files:
file_list.append(rel_path + "/" + item)
for item in dirs:
dir_list.append(rel_path + "/" + item)
# print ("Dirs: ")
# print("\n".join(dir_list))
# print("Files: ")
# print("\n".join(file_list))
# setting the needed component.
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
self.jinja_generator_ = JinjaGenerator()
dir_template_spec = template_path + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.set_empty_spec():
msg_err = "Failed generating empty spec"
self.log_error(msg_err)
return False
if not self.file_generator_.configure(self.xml_parser_, self.spec_):
return False
if not self.jinja_generator_.configure(self.xml_parser_, self.spec_):
return False
is_ok = True
for item in file_list:
self.log("Checking file: {}".format(item))
item_abs = path_root_template + "/" + item
is_ok = self.check_template_file(item_abs)
if not is_ok:
break
if is_ok:
self.log("No error detected")
else:
self.log_error("Revise the template")
return is_ok
def generate_package(self, package_desc, output_path):
"""launches the package generation
Args:
package_desc (str): xml file containing the package description
output_path (str): directory into which the package is created
Returns:
Bool: True if the operation succeeded
"""
if not os.path.exists(output_path):
msg_err = "Incorrect desired package path ({})".format(output_path)
self.log_error(msg_err)
return False
if not os.path.isdir(output_path):
msg_err = "Desired package path ({}) not a directory ".format(
output_path)
self.log_error(msg_err)
return False
# Initialising needed components
# todo bring it to the constructor?
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
self.jinja_generator_ = JinjaGenerator()
# Start finding the template
template = self.xml_parser_.get_template(package_desc)
if template is None:
return False
# Locate template location
try:
[all_template_path, template_names] = self.get_template_info()
except rospkg.common.ResourceNotFound as error:
msg = "Package package_generator_templates not found in rospack"
self.log_error(msg)
self.log_error(error)
return False
except OSError as error:
msg = "No template dounf in package_generator_templates"
self.log_error(msg)
self.log_error(error)
return False
if template not in template_names:
msg = "Template requested: {} unknown".format(template)
self.log_error(msg)
msg = "Available templates: {}".format(template_names)
self.log_error(msg)
return False
template_path = all_template_path + "/" + template
# confirm this is a template...
if not self.check_template_structure(template_path):
msg = "Please revise template structure"
self.log_error(msg)
return False
# template localized, ready to work!
self.template_path_ = template_path
self.path_pkg_backup_ = None
dir_template_spec = self.template_path_ + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.load(package_desc):
msg_err = "Prb while parsing xml file {}".format(package_desc)
self.log_error(msg_err)
return False
# todo why only the custom generator is configured?
if not self.file_generator_.configure(self.xml_parser_, self.spec_):
return False
package_name = self.xml_parser_.get_package_spec()["name"]
self.package_path_ = output_path + "/" + package_name
if os.path.exists(self.package_path_):
self.log_warn("Package {} already exists".format(
self.package_path_))
# moving preexisting code.
# generating dir name using date
now = datetime.datetime.now()
str_date = now.strftime("%Y_%m_%d_%H_%M_%S")
self.path_pkg_backup_ = "/tmp/{}_{}".format(
os.path.basename(self.package_path_), str_date)
self.log_warn("Original package temporally stored in {}".format(
self.path_pkg_backup_))
# TODO check if the move succeeded
shutil.move(self.package_path_, self.path_pkg_backup_)
else:
self.log("Package to be created in {}".format(self.package_path_))
os.makedirs(self.package_path_)
nb_comp = self.xml_parser_.get_number_comps()
self.log("Number of components defined: {}".format(nb_comp))
if not self.generate_content():
return False
# we store the model into the directory model
path = self.package_path_ + "/model"
if not os.path.exists(path):
os.makedirs(path)
path += "/" + package_name + ".ros_package"
if self.xml_parser_.is_dependency_complete_:
try:
if os.path.abspath(package_desc) == os.path.abspath(path):
# self.log_warn("Using generated model...")
stored_desc = self.path_pkg_backup_ + "/model/" + package_name + ".ros_package"
# self.log("check {} is absolute: {}".format(package_desc, os.path.abspath(package_desc)))
shutil.copyfile(stored_desc, path)
else:
shutil.copyfile(package_desc, path)
self.log("Package model saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
else:
# some dependencies were automatically added
# model needs to be rewritten
try:
self.xml_parser_.write_xml(path)
self.log("Package model updated & saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
is_ok = self.handle_maintained_files()
return is_ok
class CodeGeneratorTest(unittest.TestCase):
"""
tests for the code generator
"""
def setUp(self):
"""
Common initialization for all tester
"""
file_content = (
'<?xml version="1.0" encoding="UTF-8"?>'
'\n'
'<package name="great_package" author="anthony" author_email="*****@*****.**"'
'\n'
' description="The extended package" license="TODO" copyright="TRI">'
'\n'
' <component name="node_extended" frequency="100.0">'
'\n'
' <publisher name="pub" type="std_msgs::Bool" description="a description"/>'
'\n'
' <publisher name="pub_second" type="std_msgs::String" description="a description"/>'
'\n'
' <subscriber name="sub_in" type="std_msgs::String" description="a description" />'
'\n'
' <serviceClient name="service_client" type="std_srvs::Trigger" description="a description"/>'
'\n'
' <serviceServer name="service_server" type="std_srvs::SetBool" description="a description"/>'
'\n'
' <parameter name="param_one" type="std::string" value="Empty" description="a description"/>'
'\n'
' <parameter name="param_two" type="bool" value="1" description="a description"/>'
'\n'
' <actionServer name="action_server" type="bride_tutorials::TriggerPublish"'
'\n'
' description="a description"/>'
'\n'
' <actionClient name="action_client" type="bride_tutorials::TriggerPublish"'
'\n'
' description="a description"/>'
'\n'
' </component>'
'\n'
'<depend>std_msgs</depend>'
'\n'
'<depend>std_srvs</depend>'
'\n'
'<depend>bride_tutorials</depend>'
'\n'
'<depend>roscpp</depend>'
'\n'
'<depend>actionlib</depend>'
'\n'
'<depend>actionlib_msgs</depend>'
'\n'
'</package>'
'\n')
# print "File content: \n{}".format(file_content)
self.dir_name = "/tmp/test_package_generator"
if not os.path.exists(self.dir_name):
print "Creating the repo {}".format(self.dir_name)
os.makedirs(self.dir_name)
file_xml = self.dir_name + "/node_spec.ros_package"
with open(file_xml, 'w') as open_file:
open_file.write(file_content)
rospack = rospkg.RosPack()
self.template_path_ = rospack.get_path('package_generator_templates')
self.template_path_ += "/templates/cpp_node_update/"
self.spec = TemplateSpec()
self.assertTrue(self.spec.load_spec(self.template_path_ + "config"))
self.xml_parser = PackageXMLParser()
self.assertTrue(self.xml_parser.set_template_spec(self.spec))
self.assertTrue(self.xml_parser.load(file_xml))
self.generator_ = CodeGenerator()
self.assertTrue(self.generator_.configure(self.xml_parser, self.spec))
self.generator_.reset_output_file()
def test_apply_function(self):
"""
To apply a function on a tag
"""
filename = self.dir_name + "/template_test_apply_function.cpp"
file_content = (
'{forallpublisher}'
'\n'
'name={name}'
'\n'
'pkg={apply-get_package_type} '
'\n'
'type={apply-get_class_type}'
'\n'
'in one line: {name}: {apply-get_python_type}'
'\n'
'cpp path: {name}: {apply-get_cpp_path}'
'\n'
'more tricky: {name}: {apply-get_cpp_path} {unknowntag}'
'\n'
'more tricky: {name}: {apply-get_cpp_path} {apply-unknownname}'
'\n'
'\n'
'{endforallpublisher}')
expected_output = (
"name=pub"
"\n"
"pkg=std_msgs "
"\n"
"type=Bool"
"\n"
"in one line: pub: std_msgs.Bool"
"\n"
"cpp path: pub: std_msgs/Bool"
"\n"
"more tricky: pub: std_msgs/Bool {unknowntag}"
"\n"
"more tricky: pub: std_msgs/Bool {apply-unknownname}"
"\n"
""
"\n"
"name=pub_second"
"\n"
"pkg=std_msgs "
"\n"
"type=String"
"\n"
"in one line: pub_second: std_msgs.String"
"\n"
"cpp path: pub_second: std_msgs/String"
"\n"
"more tricky: pub_second: std_msgs/String {unknowntag}"
"\n"
"more tricky: pub_second: std_msgs/String {apply-unknownname}"
"\n")
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(self.generator_.process_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
# print "Comparing |{}| with |{}|".format(generated, groundtruth)
self.assertEqual(generated, groundtruth)
def test_multi_line(self):
"""
Test the conversion of multi-line
"""
filename = self.dir_name + "/template_test_multiple_line.cpp"
file_content = ('hello {componentName}'
'\n'
'{forallpublisher}'
'\n'
'if (component_data_.{name}_active)'
'\n'
' pub_.publish(component_data_.{name});'
'\n'
'{endforallpublisher}'
'\n')
expected_output = ("hello node_extended"
"\n"
"if (component_data_.pub_active)"
"\n"
" pub_.publish(component_data_.pub);"
"\n"
"if (component_data_.pub_second_active)"
"\n"
" pub_.publish(component_data_.pub_second);"
"\n")
with open(filename, 'w') as open_file:
open_file.write(file_content)
self.assertTrue(self.generator_.process_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
self.assertEqual(generated, groundtruth)
def test_multi_line_bracket(self):
"""
Test correct management of multi-line with bracket
"""
filename = self.dir_name + "/template_test_multiple_line.cpp"
file_content = ('hello {componentName}'
'\n'
'{forallpublisher}'
'\n'
'if (component_data_.{name}_active)'
'\n'
'{'
'\n'
' pub_.publish(component_data_.{name});'
'\n'
'}'
'\n'
'{endforallpublisher}'
'\n')
expected_output = ("hello node_extended"
"\n"
"if (component_data_.pub_active)"
"\n"
"{"
"\n"
" pub_.publish(component_data_.pub);"
"\n"
"}"
"\n"
"if (component_data_.pub_second_active)"
"\n"
"{"
"\n"
" pub_.publish(component_data_.pub_second);"
"\n"
"}"
"\n")
with open(filename, 'w') as open_file:
open_file.write(file_content)
self.assertTrue(self.generator_.process_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
self.assertEqual(generated, groundtruth)
def test_multi_tag_single_line(self):
"""
todo An interesting reference to look at if needed
http://stackoverflow.com/questions/17215400/python-format-string-unused-named-arguments
"""
filename = self.dir_name + "/template_test_multiple_tag.cpp"
file_content = (
'Let us start !!!!'
'\n'
'hello {componentName}'
'\n'
' void update({componentName}_data &data, {componentName}_config config)'
'\n'
'Bye Bye'
'\n')
expected_output = (
"Let us start !!!!"
"\n"
"hello node_extended"
"\n"
" void update(node_extended_data &data, node_extended_config config)"
"\n"
"Bye Bye"
"\n")
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(self.generator_.process_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
self.assertEqual(generated, groundtruth)
def test_if_condition(self):
"""
@brief Test the if condition
@param self The object
@return nothing
"""
filename = self.dir_name + "/template_test_multiple_line.cpp"
file_content = ('Let us start !!!!'
'\n'
'{ifparameter}'
'\n'
'<build_depend>dynamic_reconfigure</build_depend>'
'\n'
'<run_depend>dynamic_reconfigure</run_depend>'
'\n'
'{endifparameter}'
'\n'
'{ifactionServer}'
'\n'
'<build_depend>actionlib</build_depend>'
'\n'
'<run_depend>actionlib</run_depend>'
'\n'
'{endifactionServer}'
'\n')
expected_output = ("Let us start !!!!"
"\n"
"<build_depend>dynamic_reconfigure</build_depend>"
"\n"
"<run_depend>dynamic_reconfigure</run_depend>"
"\n"
"<build_depend>actionlib</build_depend>"
"\n"
"<run_depend>actionlib</run_depend>"
"\n")
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(self.generator_.process_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
self.assertEqual(generated, groundtruth)
def test_complete_ros(self):
"""
@brief Test the generation of a complete cpp ros file
@param self The object
@return nothing
"""
filename = self.template_path_ + 'template/ros/src/component_ros.cpp'
self.assertTrue(self.generator_.process_file(filename))
output_file = self.dir_name + "/component_ros.cpp"
self.assertTrue(self.generator_.write_rendered_file(output_file))
def test_complete_cmake(self):
"""
@brief Test generation of a complete cmake
@param self The object
@return { description_of_the_return_value }
"""
filename = self.template_path_ + 'template/CMakeLists.txt'
self.assertTrue(self.generator_.process_file(filename))
output_file = self.dir_name + "/CMakeLists.txt"
self.assertTrue(self.generator_.write_rendered_file(output_file))
def test_complete_readme(self):
"""
@brief Test generation of a complete readme
@param self The object
@return { description_of_the_return_value }
"""
filename = self.template_path_ + 'template/README.md'
self.assertTrue(self.generator_.process_file(filename))
output_file = self.dir_name + "/README.md"
self.assertTrue(self.generator_.write_rendered_file(output_file))
def test_complete_package(self):
"""
@brief Test the generation of a complete package
@param self The object
@return nothing
"""
filename = self.template_path_ + 'template/package.xml'
self.assertTrue(self.generator_.process_file(filename))
output_file = self.dir_name + "/package.xml"
self.assertTrue(self.generator_.write_rendered_file(output_file))
def main_generate_xml():
"""Generate a xml package description based on a template spec
Returns:
int: negative value on error
"""
rospack = rospkg.RosPack()
try:
node_path = rospack.get_path('package_generator_templates')
default_templates_path = node_path + "/templates/"
except rospkg.common.ResourceNotFound as error:
msg = "Package package_generator_templates not found in rospack"
print colored(msg, "yellow")
print colored("{}".format(error), "yellow")
default_templates_path = None
available_templates = None
# look for the templates available
if default_templates_path is not None:
available_templates = os.listdir(default_templates_path)
if len(sys.argv) != 3:
print colored("Wrong input parameters !", "red")
print colored(USAGE, "yellow")
if available_templates is not None:
msg = "Available templates are: {}"
print colored(msg.format(available_templates), 'yellow')
print "Bye bye"
return -1
path_template = sys.argv[1]
package_spec = sys.argv[2]
# check first if the provided filename is correct
# we do not accept the name to be an existing file
if os.path.isfile(package_spec):
print colored("file {} already exists".format(package_spec), "red")
print colored("Please select another filename, or remove the file")
print "Bye Bye"
return -1
# if relative path or absolute path, check the containing folder exists
folder = os.path.dirname(package_spec)
if folder and not os.path.isdir(folder):
msg_err = "file {} cannot be created in {}".format(
package_spec, folder)
print colored(msg_err, "red")
print colored("Please adjust parameter")
print "Bye Bye"
return -1
# searching for the template location
if os.path.isabs(path_template):
print "Loading model from absolute path {}".format(path_template)
else:
# relative path.
# either from the current path, or from the template package
path_current = os.getcwd()
path_attempt = path_current + "/" + path_template
if os.path.isdir(path_attempt):
path_template = path_attempt
print "Loading template from path {}".format(path_template)
else:
if path_template in available_templates:
path_template = default_templates_path + path_template
msg = "Loading template from template package: {}"
print msg.format(path_template)
else:
msg = "Template name not found in package_generator_templates"
print colored(msg, "red")
print colored("Please verify your setup", "red")
return -1
package_parser = PackageXMLParser()
dir_template_spec = path_template + "/config/"
spec = TemplateSpec()
if not spec.load_spec(dir_template_spec):
print colored("Could not load the template spec", "red")
return -1
# print "Setting the template spec to \n {}".format(spec.dico_)
if not package_parser.set_template_spec(spec):
print colored("Prb while setting the parser dictionary", "red")
return -1
package_parser.generate_xml_from_spec(package_spec)
print colored("Bye bye", "green")
return 0
class JinjaGeneratorTest(unittest.TestCase):
"""
tests for the code generator
"""
def setUp(self):
"""
Common initialization for all tester
"""
file_content = (
'<?xml version="1.0" encoding="UTF-8"?>' '\n'
'<package name="great_package" author="anthony" author_email="*****@*****.**"' '\n'
' description="The extended package" license="TODO" copyright="TRI">' '\n'
' <component name="node_extended" frequency="100.0">' '\n'
' <publisher name="pub" type="std_msgs::Bool" description="a description"/>' '\n'
' <publisher name="pub_second" type="std_msgs::String" description="a description"/>' '\n'
' <subscriber name="sub_in" type="std_msgs::String" description="a description" />' '\n'
' <serviceClient name="service_client" type="std_srvs::Trigger" description="a description"/>' '\n'
' <serviceServer name="service_server" type="std_srvs::SetBool" description="a description"/>' '\n'
' <parameter name="param_one" type="std::string" value="Empty" description="a description"/>' '\n'
' <parameter name="param_two" type="bool" value="1" description="a description"/>' '\n'
' <actionServer name="action_server" type="bride_tutorials::TriggerPublish"' '\n'
' description="a description"/>' '\n'
' <actionClient name="action_client" type="bride_tutorials::TriggerPublish"' '\n'
' description="a description"/>' '\n'
' </component>' '\n'
'<depend>std_msgs</depend>' '\n'
'<depend>std_srvs</depend>' '\n'
'<depend>bride_tutorials</depend>' '\n'
'<depend>actionlib</depend>' '\n'
'<depend>actionlib_msgs</depend>' '\n'
'<depend>roscpp</depend>' '\n'
'</package>' '\n')
# print "File content: \n{}".format(file_content)
self.dir_name = "/tmp/test_package_generator"
if not os.path.exists(self.dir_name):
print "Creating the repo {}".format(self.dir_name)
os.makedirs(self.dir_name)
file_xml = self.dir_name + "/node_spec.ros_package"
with open(file_xml, 'w') as open_file:
open_file.write(file_content)
rospack = rospkg.RosPack()
self.template_path_ = rospack.get_path('package_generator_templates')
self.template_path_ += "/templates/cpp_node_update/"
self.spec = TemplateSpec()
self.assertTrue(self.spec.load_spec(self.template_path_ + "config"))
self.xml_parser = PackageXMLParser()
self.assertTrue(self.xml_parser.set_template_spec(self.spec))
self.assertTrue(self.xml_parser.load(file_xml))
self.generator_ = JinjaGenerator()
self.assertTrue(self.generator_.configure(self.xml_parser, self.spec))
def test_is_launching(self):
"""
Confirm that Jinja does not interfere with custom tags
"""
filename = self.dir_name + "/template_test_apply_function.cpp"
file_content = (
'{forallpublisher}' '\n'
'name={name}' '\n'
'pkg={apply-get_package_type} ' '\n'
'type={apply-get_class_type}' '\n'
'in one line: {name}: {apply-get_python_type}' '\n'
'cpp path: {name}: {apply-get_cpp_path}' '\n'
'more tricky: {name}: {apply-get_cpp_path} {unknowntag}' '\n'
'more tricky: {name}: {apply-get_cpp_path} {apply-unknown}' '\n'
'\n'
'{endforallpublisher}')
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(self.generator_.generate_disk_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
file_content.splitlines()):
# print "Comparing |{}| with |{}|".format(generated, groundtruth)
self.assertEqual(generated, groundtruth)
def test_jinja(self):
"""
General test
"""
filename = self.dir_name + "/template_test_apply_function.cpp"
file_content = (
'{forallpublisher}' '\n'
'name={{active_comp}}' '\n'
'pkg={{package.name}}' '\n'
'{% for item in components[0].interface.publisher %}' '\n'
'publisher: {{ item.name }}' '\n'
'{% endfor %}' '\n'
'{% if components[0].interface.publisher %}' '\n'
'At least one publisher ' '\n'
'{% endif %}' '\n'
'{% if components[0].interface.directPublisher %}' '\n'
'At least one direct publisher ' '\n'
'{% else %}' '\n'
'No direct publisher ' '\n'
'{% endif %}' '\n'
'{endforallpublisher}' '\n')
expected_output = (
'{forallpublisher}' '\n'
'name=0' '\n'
'pkg=great_package' '\n'
'publisher: pub' '\n'
'publisher: pub_second' '\n'
'At least one publisher ' '\n'
'No direct publisher ' '\n'
'{endforallpublisher}')
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(self.generator_.generate_disk_file(filename))
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
self.assertEqual(generated, groundtruth)
def test_custom_on_jinja(self):
"""to be completed
"""
filename = self.dir_name + "/template_test_apply_function.cpp"
file_content = (
'{ifpublisher}' '\n'
'name={{active_comp}}' '\n'
'pkg={{package.name}}' '\n'
'{% for item in components[0].interface.publisher %}' '\n'
'publisher: {{ item.name }}' '\n'
'{% endfor %}' '\n'
'' '\n'
'{% if components[0].interface.publisher %}' '\n'
'At least one publisher ' '\n'
'{% endif %}' '\n'
'{% if components[0].interface.directPublisher %}' '\n'
'At least one direct publisher ' '\n'
'{% else %}' '\n'
'No direct publisher ' '\n'
'{% endif %}' '\n'
'{endifpublisher}' '\n')
expected_output = (
'name=0' '\n'
'pkg=great_package' '\n'
'publisher: pub' '\n'
'publisher: pub_second' '\n'
'' '\n'
'At least one publisher ' '\n'
'No direct publisher ' '\n')
custom_generator = CodeGenerator()
self.assertTrue(custom_generator.configure(self.xml_parser, self.spec))
custom_generator.reset_output_file()
with open(filename, 'w') as openfile:
openfile.write(file_content)
self.assertTrue(custom_generator.process_file(filename))
# print "Custom generation \n"
# for line in custom_generator.rendered_:
# print line
self.assertTrue(self.generator_.generate_open_file(custom_generator.rendered_))
# print "Rendered file \n"
# print self.generator_.rendered_
for generated, groundtruth in zip(self.generator_.rendered_,
expected_output.splitlines()):
# print "Comparing |{}| with |{}|".format(generated, groundtruth)
self.assertEqual(generated, groundtruth)
def test_write_file(self):
file_content = (
'name={{active_comp}}' '\n'
'pkg={{package.name}}' '\n'
'{% for item in components[0].interface.publisher %}' '\n'
'publisher: {{ item.name }}' '\n'
'{% endfor %}' '\n'
'{% if components[0].interface.publisher %}' '\n'
'At least one publisher ' '\n'
'{% endif %}' '\n'
'{% if components[0].interface.directPublisher %}' '\n'
'At least one direct publisher ' '\n'
'{% else %}' '\n'
'No direct publisher ' '\n'
'{% endif %}')
expected_output = (
'name=0' '\n'
'pkg=great_package' '\n'
'publisher: pub' '\n'
'publisher: pub_second' '\n'
'At least one publisher ' '\n'
'No direct publisher ' '\n')
template_name = self.dir_name + "/jinja_template.txt"
with open(template_name, 'w') as openfile:
openfile.write(file_content)
rendered_name = self.dir_name + "/jinja_rendered.txt"
self.assertTrue(self.generator_.generate_disk_file(template_name, rendered_name))
self.assertTrue(os.path.isfile(rendered_name))
lines = list()
with open(rendered_name) as input_file:
for line in input_file:
lines.append(line.rstrip('\n'))
for generated, groundtruth in zip(lines,
expected_output.splitlines()):
# print "Comparing |{}| with |{}|".format(generated, groundtruth)
self.assertEqual(generated, groundtruth)
class PackageGenerator(EnhancedObject):
"""Handle the genration of a whole package
Attributes:
file_generator_ (TYPE): Description
package_path_ (TYPE): Description
path_pkg_backup_ (TYPE): Description
spec_ (TemplateSpec): configuration of the template model
template_path_ (TYPE): Description
xml_parser_ (TYPE): Description
"""
def __init__(self, name="PackageGenerator"):
""" Intialisation of the object
Args:
name (str, optional): Name of the component, for printing aspect
"""
# call super class constructor
super(PackageGenerator, self).__init__(name)
# path to the template to use
self.template_path_ = None
# base location of the package to create
self.package_path_ = None
# parser of the package description
self.xml_parser_ = None
# config parameter provide with the template
self.spec_ = None
# generic file generator
self.file_generator_ = None
# if the package already existed, location of the package backup
self.path_pkg_backup_ = None
def set_package_template(self, template_path):
"""set the package template
Args:
template_path (str): path to the package template
Returns:
Bool: True if basic sanity checks are done with success
"""
if not os.path.exists(template_path):
msg = "Template path ({}) is incorrect ".format(template_path)
self.log_error(msg)
return False
if not os.path.isdir(template_path):
msg = "Template path ({}) is not a directory ".format(
template_path)
self.log_error(msg)
return False
# check if minimum information is present.
details = """A template should contain:
* config/dictionary.yaml : the dictionary to be used
* [optional] config/functions.py : additional functions used in the generation
* template/* set of elements to be generated
Revise the template, and compare to examples
"""
is_ok = True
# check for directories
required_folders = ["config", "template"]
for item in required_folders:
req_folder = template_path + "/" + item
if not os.path.isdir(req_folder):
msg_err = "Error \n Expecting to have folder " + item
msg_err += " in " + template_path
self.log_error(msg_err)
is_ok = False
# check for files
required_files = ["config/dictionary.yaml"]
for item in required_files:
req_file = template_path + "/" + item
if not os.path.isfile(req_file):
msg_err = "Error.\n Expecting to have file " + item
msg_err += " in " + template_path
self.log_error(msg_err)
is_ok = False
if not is_ok:
self.log_error("\n{}".format(details))
return False
self.template_path_ = template_path
return True
def generate_package(self, package_desc, output_path):
"""launches the package generation
Args:
package_desc (str): xml file containing the package description
output_path (str): directory into which the package is created
Returns:
Bool: True if the operation succeeded
"""
self.path_pkg_backup_ = None
if not os.path.exists(output_path):
msg_err = "Template path ({}) is incorrect ".format(output_path)
self.log_error(msg_err)
return False
if not os.path.isdir(output_path):
msg_err = "Template path ({}) not a directory ".format(output_path)
self.log_error(msg_err)
return False
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
dir_template_spec = self.template_path_ + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.load(package_desc):
msg_err = "Prb while parsing xml file {}".format(package_desc)
self.log_error(msg_err)
return False
self.file_generator_.configure(self.xml_parser_, self.spec_)
package_name = self.xml_parser_.get_package_spec()["name"]
self.package_path_ = output_path + "/" + package_name
if os.path.exists(self.package_path_):
self.log_warn("Package {} already exists".format(
self.package_path_))
# moving preexisting code.
# generating dir name using date
now = datetime.datetime.now()
str_date = now.strftime("%Y_%m_%d_%H_%M_%S")
self.path_pkg_backup_ = "/tmp/{}_{}".format(
os.path.basename(self.package_path_), str_date)
self.log_warn("Original package temporally stored in {}".format(
self.path_pkg_backup_))
# TODO check if the move succeeded
shutil.move(self.package_path_, self.path_pkg_backup_)
else:
self.log("Package to be created in {}".format(self.package_path_))
os.makedirs(self.package_path_)
package_content = os.listdir(self.template_path_ + "/template")
nb_node = self.xml_parser_.get_number_nodes()
self.log("Number of nodes defined: {}".format(nb_node))
# launching the file generation, except common files
self.log("Generating files specific to nodes")
is_ok = True
for id_node in range(nb_node):
# self.log_error("Handling node {}".format(id_node))
if not self.xml_parser_.set_active_node(id_node):
is_ok = False
break
node_name = self.xml_parser_.data_node_[id_node]["attributes"][
"name"]
self.log_warn("Handling files for node {}".format(node_name))
# reconfiguring the generator to adjust to the new active node
self.file_generator_.configure(self.xml_parser_, self.spec_)
for item in package_content:
file_generic = self.template_path_ + '/template/' + item
is_ok = self.generate_content(file_generic, False)
if not is_ok:
break
if not is_ok:
return False
self.log("Generating files common to all nodes")
is_ok = True
# todo should we do something with the active_node file?
for item in package_content:
file_generic = self.template_path_ + '/template/' + item
is_ok = self.generate_content(file_generic, True)
if not is_ok:
break
if not is_ok:
return False
# we store the model into the directory model
path = self.package_path_ + "/model"
if not os.path.exists(path):
os.makedirs(path)
path += "/" + package_name + ".ros_package"
if self.xml_parser_.is_dependency_complete_:
try:
if os.path.abspath(package_desc) == os.path.abspath(path):
# self.log_warn("Using generated model...")
stored_desc = self.path_pkg_backup_ + "/model/" + package_name + ".ros_package"
# self.log("check {} is absolute: {}".format(package_desc, os.path.abspath(package_desc)))
shutil.copyfile(stored_desc, path)
else:
shutil.copyfile(package_desc, path)
self.log("Package model saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
else:
# some dependencies were automatically added
# model needs to be rewritten
try:
self.xml_parser_.write_xml(path)
self.log("Package model updated & saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
is_ok = self.handle_maintained_files()
return is_ok
def handle_maintained_files(self):
""" Restore file Developer requests to maintain
Assuming these patterns are defined in file .gen_maintain
Returns:
Bool: True on sucess
"""
# check for files to be maintained
if self.path_pkg_backup_ is None:
# package just created, no maintained file
return True
filename_rel = ".gen_maintain"
filename_abs = self.path_pkg_backup_ + "/" + filename_rel
if os.path.exists(filename_abs) and os.path.isfile(filename_abs):
self.log("Checking content to maintain after update")
else:
self.log("no maintained file defined in previous package version")
return True
with open(filename_abs) as open_file:
for line in open_file:
line = line.rstrip('\n')
if not line:
continue
path_abs = self.path_pkg_backup_ + "/" + line
if not os.path.exists(path_abs):
msg = "Content {} not found. Revise {} content"
self.log_error(msg.format(line, filename_abs))
continue
new_path = self.package_path_ + "/" + line
if os.path.isfile(path_abs):
try:
self.log("Restoring file {}".format(line))
# check if directories needs to be created
dirname = os.path.dirname(line)
if dirname:
if not os.path.isdir(path_abs):
os.makedirs(self.package_path_ + "/" + dirname)
shutil.copyfile(path_abs, new_path)
except IOError as error:
msg = "Could not restore a file: {}"
self.log_error(msg.format(error))
continue
if os.path.isdir(path_abs):
try:
self.log("Restoring folder {}".format(line))
shutil.copytree(path_abs, new_path)
except IOError as error:
msg = "Could not restore folder: {}"
self.log_error(msg.format(error))
continue
self.log_error("Unkown statement {}".format(line))
# restoring the maintained content file
try:
self.log("Restoring file {}".format(filename_rel))
new_path = self.package_path_ + "/" + filename_rel
shutil.copyfile(filename_abs, new_path)
except IOError as error:
msg = "Could not restore file: {}"
self.log_error(msg.format(error))
return True
def generate_content(self, input_item, is_generic):
"""recursive function enabling the generation and storage of a file
or all content of a directory
Args:
input_item (str): path to a file or a directory to be generated
is_generic (Bool): specify if we handle generic files or node specific ones
Returns:
Todo: how to handle the file name, when several nodes are defined.
"""
is_ok = True
# get the package name
package_name = self.xml_parser_.get_package_spec()["name"]
rel_path = os.path.relpath(input_item,
self.template_path_ + "/template")
# check if the name 'package_name is present in the name'
rel_path = rel_path.replace('package_name', package_name)
output_item = self.package_path_ + '/' + rel_path
if os.path.isdir(input_item):
# self.log("Handling folder {}".format(rel_path))
if os.path.exists(output_item):
# self.log_warn("Directory {} already exists".format(output_item))
pass
else:
# self.log("Creating directory {}".format(rel_path))
os.makedirs(output_item)
for item in os.listdir(input_item):
is_ok = self.generate_content(input_item + '/' + item,
is_generic)
if not is_ok:
break
return is_ok
# file to be generated.
# we check if "node" is in the filename (ie not a generic file).
basename = os.path.basename(rel_path)
if 'node' in basename:
# this is a node specific file. Flag is_generic defines our strategy
if is_generic:
# nothing to be done, we just skip it
return True
else:
# this is a generic file, common to all nodes
if not is_generic:
# we do nothing
return True
self.log("Handling template file {}".format(rel_path))
# The only difference between generic and no generic is that
# in the generic case we have to instanciate the node name
if 'node' in basename:
# This is a node specific file. We adjust the filename
basename = basename.replace(
'node',
self.xml_parser_.get_active_node_spec()["attributes"]["name"])
# the rest is common
rel_path = os.path.join(os.path.dirname(rel_path), basename)
output_item = os.path.join(os.path.dirname(output_item), basename)
# Normally an empty file should not be written
# The exception is currently only for the special python file __init__.py
is_write_forced = (os.path.basename(output_item) == '__init__.py')
if self.path_pkg_backup_ is None:
self.log("Generating file {} in {}".format(rel_path, output_item))
is_ok = self.file_generator_.generate_file(
input_item, output_item, force_write=is_write_forced)
return self.handle_status_and_advise(input_item, output_item,
is_ok)
# A previous version of the package exists
# Checking if an update is necessary
previous_filename = os.path.join(self.path_pkg_backup_, rel_path)
# Check 1: does this file exist?
if not os.path.isfile(previous_filename):
msg = "File {} not previously existing. Just write it"
self.log_warn(msg.format(rel_path))
is_ok = self.file_generator_.generate_file(input_item, output_item,
is_write_forced)
return self.handle_status_and_advise(input_item, output_item,
is_ok)
# File already existing. Processing previous version
is_update_needed = False
file_analyzor = GeneratedFileAnalysis()
is_ok = file_analyzor.extract_protected_region(previous_filename)
if is_ok:
# Check if Developer inserted any contribution
if file_analyzor.extracted_areas_:
# contribution found, merge needed
is_update_needed = True
else:
self.log("No Developer contribution found")
else:
msg = "prb while extracting protected area in {}"
self.log_error(msg.format(previous_filename))
self.log_error("Previous file to be manually merged, sorry")
# now we know if an update is needed
if is_ok and is_update_needed:
# self.log("Updating file {} in {}".format(rel_path, output_item))
self.log("Updating file {}".format(rel_path))
is_ok = self.file_generator_.generate_file(input_item)
if is_ok:
if self.file_generator_.get_len_gen_file() == 0:
msg = "New generated file empty. No code maintained from previous version"
self.log_warn(msg)
# we write it if forced
if is_write_forced:
is_ok = self.file_generator_.write_output_file(
output_item)
else:
self.log("Merging with previous version")
self.file_generator_.tmp_buffer_ = file_analyzor.update_file(
self.file_generator_.tmp_buffer_)
is_ok = self.file_generator_.write_output_file(output_item)
return self.handle_status_and_advise(input_item, output_item,
is_ok)
# Although the file existed before, we do not have to maintain it
is_ok = self.file_generator_.generate_file(input_item, output_item,
is_write_forced)
return self.handle_status_and_advise(input_item, output_item, is_ok)
def handle_status_and_advise(self, input_file, output_file, gen_flag):
"""Depending on the file generation process outcome,
Adjust file status and inform user
Args:
input_file (str): path of the template file used
output_file (TYPE): path of the generated file
gen_flag (Bool): Success of the generation process
Returns:
Bool: True on success of the file generation
"""
if not gen_flag:
msg = "Prb while generating file {}".format(output_file)
self.log_error(msg)
return False
# so the file generation went well
if self.file_generator_.get_len_gen_file() == 0:
# Only file __init__.py is kept empty
if os.path.basename(output_file) != '__init__.py':
msg = "File {} not written since empty".format(output_file)
self.log_warn(msg)
self.log_warn("Check: {}".format(
os.path.basename(output_file)))
return True
# file has content
file_status = os.stat(input_file)
os.chmod(output_file, file_status.st_mode)
self.log("File {} handled".format(input_file))
self.log("*********************************")
return True
def generate_package(self, package_desc, output_path):
"""launches the package generation
Args:
package_desc (str): xml file containing the package description
output_path (str): directory into which the package is created
Returns:
Bool: True if the operation succeeded
"""
self.path_pkg_backup_ = None
if not os.path.exists(output_path):
msg_err = "Template path ({}) is incorrect ".format(output_path)
self.log_error(msg_err)
return False
if not os.path.isdir(output_path):
msg_err = "Template path ({}) not a directory ".format(output_path)
self.log_error(msg_err)
return False
self.spec_ = TemplateSpec()
self.xml_parser_ = PackageXMLParser()
self.file_generator_ = CodeGenerator()
dir_template_spec = self.template_path_ + "/config/"
if not self.spec_.load_spec(dir_template_spec):
self.log_error("Could not load the template spec")
return False
if not self.xml_parser_.set_template_spec(self.spec_):
msg_err = "Package spec not compatible with xml parser expectations"
self.log_error(msg_err)
return False
if not self.xml_parser_.load(package_desc):
msg_err = "Prb while parsing xml file {}".format(package_desc)
self.log_error(msg_err)
return False
self.file_generator_.configure(self.xml_parser_, self.spec_)
package_name = self.xml_parser_.get_package_spec()["name"]
self.package_path_ = output_path + "/" + package_name
if os.path.exists(self.package_path_):
self.log_warn("Package {} already exists".format(
self.package_path_))
# moving preexisting code.
# generating dir name using date
now = datetime.datetime.now()
str_date = now.strftime("%Y_%m_%d_%H_%M_%S")
self.path_pkg_backup_ = "/tmp/{}_{}".format(
os.path.basename(self.package_path_), str_date)
self.log_warn("Original package temporally stored in {}".format(
self.path_pkg_backup_))
# TODO check if the move succeeded
shutil.move(self.package_path_, self.path_pkg_backup_)
else:
self.log("Package to be created in {}".format(self.package_path_))
os.makedirs(self.package_path_)
package_content = os.listdir(self.template_path_ + "/template")
nb_node = self.xml_parser_.get_number_nodes()
self.log("Number of nodes defined: {}".format(nb_node))
# launching the file generation, except common files
self.log("Generating files specific to nodes")
is_ok = True
for id_node in range(nb_node):
# self.log_error("Handling node {}".format(id_node))
if not self.xml_parser_.set_active_node(id_node):
is_ok = False
break
node_name = self.xml_parser_.data_node_[id_node]["attributes"][
"name"]
self.log_warn("Handling files for node {}".format(node_name))
# reconfiguring the generator to adjust to the new active node
self.file_generator_.configure(self.xml_parser_, self.spec_)
for item in package_content:
file_generic = self.template_path_ + '/template/' + item
is_ok = self.generate_content(file_generic, False)
if not is_ok:
break
if not is_ok:
return False
self.log("Generating files common to all nodes")
is_ok = True
# todo should we do something with the active_node file?
for item in package_content:
file_generic = self.template_path_ + '/template/' + item
is_ok = self.generate_content(file_generic, True)
if not is_ok:
break
if not is_ok:
return False
# we store the model into the directory model
path = self.package_path_ + "/model"
if not os.path.exists(path):
os.makedirs(path)
path += "/" + package_name + ".ros_package"
if self.xml_parser_.is_dependency_complete_:
try:
if os.path.abspath(package_desc) == os.path.abspath(path):
# self.log_warn("Using generated model...")
stored_desc = self.path_pkg_backup_ + "/model/" + package_name + ".ros_package"
# self.log("check {} is absolute: {}".format(package_desc, os.path.abspath(package_desc)))
shutil.copyfile(stored_desc, path)
else:
shutil.copyfile(package_desc, path)
self.log("Package model saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
else:
# some dependencies were automatically added
# model needs to be rewritten
try:
self.xml_parser_.write_xml(path)
self.log("Package model updated & saved in: {}".format(path))
except IOError as error:
self.log_error("Could not store model file: {}".format(error))
is_ok = self.handle_maintained_files()
return is_ok
