def verify_manifest(self, expected_output, expected_configs=None):
diff = expected_output.get_diff(self.manifest.get_output())
if diff != Util.Container():
print("UNMATCHED FIELDS:{}\n".format(diff))
self.fail()
if not expected_configs:
self.assertEqual(self.manifest.get_configs(), [])
else:
config_idx = 0
actual_configs = self.manifest.get_configs()
for c in actual_configs:
generators = []
for g in c.generators:
actual_generator = Util.Container(
format=g.FORMAT_TYPE,
filename=g.get_filename()
)
if g.get_formatter():
actual_generator.formatter = g.get_formatter()
generators.append(actual_generator)
c.generators = generators
if actual_configs[config_idx].get_diff(expected_configs[config_idx]):
self.fail(msg="Config mismatch at index {}\n===Actual===\n{}===Expected===\n{}".format(
config_idx,
actual_configs[config_idx],
expected_configs[config_idx]
))
config_idx = config_idx + 1
self.assertEqual(len(expected_configs), config_idx)
def test_enums(self):
self.create_parser()
self.parse_lines(
":config FOO_LOG_TYPE",
" type ENUM",
" title \"Logging\"",
" default FOO_LOG_NONE",
" choice FOO_LOG_NONE \"None\"",
" choice FOO_LOG_STD \"Write to stdout/stderr\"",
" count FOO_LOG_COUNT",
":end",
":config FOO_OS_TYPE",
" type ENUM",
" choice FOO_OS_WINDOWS",
" choice FOO_OS_POSIX",
" choice FOO_OS_CUSTOM",
":end",
)
self.verify_deftree(
Util.Container(
**{
"/":
Util.Container(params=[
"id=FOO_LOG_TYPE type=ENUM title=\"Logging\" default=FOO_LOG_NONE"
+ " count=FOO_LOG_COUNT" +
" choice=FOO_LOG_NONE,\"None\"" +
" choice=FOO_LOG_STD,\"Write to stdout/stderr\"",
"id=FOO_OS_TYPE type=ENUM default=FOO_OS_WINDOWS" +
" choice=FOO_OS_WINDOWS,\"FOO_OS_WINDOWS\"" +
" choice=FOO_OS_POSIX,\"FOO_OS_POSIX\"" +
" choice=FOO_OS_CUSTOM,\"FOO_OS_CUSTOM\""
])
}))
def verify_project(self, expected):
prj = self.parser.get_target()
# Verify simple properties
self.assertIsNotNone(prj)
self.assertEqual(prj.get_name(), expected.name)
# Verify layers
for actual_layer_name, expected_layer in zip(prj.get_layer_names(),
expected.layers):
self.assertEqual(actual_layer_name, expected_layer.name)
for actual_variant_name, expected_variant in zip(
prj.get_variant_names(actual_layer_name),
expected_layer.variants):
self.assertEqual(actual_variant_name, expected_variant.name)
self.assertEqual(
prj.get_target(actual_layer_name,
actual_variant_name).filename,
expected_variant.filename)
# Verify packages
self.assertEqual(prj.get_packages(), expected.packages)
# Verify dependencies
actual_dependencies = Util.Container()
for dep_name, dependency in prj.get_dependencies():
actual_action_list = []
for action in dependency.get_actions():
actual_action_list.append(
Util.Container(id=action.ACTION_TYPE, arg=action.arg))
actual_dependencies[dep_name] = actual_action_list
self.assertEqual(Util.Container(),
actual_dependencies.get_diff(expected.dependencies))
def test_extend(self):
self.create_config_set(Util.Container(a="12", b="34"))
self.config.extend(self.new_settings(Util.Container(b="56", c="78")))
self.assertTrue(self.config.evaluate("a==12"))
self.assertTrue(self.config.evaluate("b==56"))
self.assertTrue(self.config.evaluate("c==78"))
def add_variant(self, layer_name, variant_name, filename):
"""Add a new variant into the layer"""
layer_ref = self._get_layer_ref(layer_name)
if layer_ref is not None:
variant_ref = Util.Container(name=variant_name,
filename=filename,
config=Util.Container())
layer_ref.variants.append(variant_ref)
def test_dependencies(self):
self.create_parser()
self.parse_lines(
":project DepTest",
" :dependency FOO",
" dest chu.zip",
" url https://foohub.com/foo_man/FOO/master/chu.zip",
" extract chu/",
" :end",
" :ext_package FOO extract/chu.gman",
" :dependency BAR",
" dest bar.bin",
" url https://foohub.com/foo_man/FOO/master/bar.bin",
" :end",
" :lcl_package BAR external/BAR.gman",
":end",
)
self.verify_project(
Util.Container(
**{
"name":
"DepTest",
"layers": [],
"packages": [
Util.Container(filename="extract/chu.gman",
file_root=Project.ROOT.DEPENDENCY,
module_root=Project.ROOT.SOURCE,
module_id="FOO"),
Util.Container(filename="external/BAR.gman",
file_root=Project.ROOT.SOURCE,
module_root=Project.ROOT.DEPENDENCY,
module_id="BAR")
],
"dependencies":
Util.Container(
**{
"FOO": [
Util.Container(id="dest", arg="chu.zip"),
Util.Container(
id="url",
arg=
"https://foohub.com/foo_man/FOO/master/chu.zip"
),
Util.Container(id="extract", arg="chu/")
],
"BAR": [
Util.Container(id="dest", arg="bar.bin"),
Util.Container(
id="url",
arg=
"https://foohub.com/foo_man/FOO/master/bar.bin"
)
]
}),
}))
def __init__(self, configs=Util.Container(), debug_mode=False):
Log.Debuggable.__init__(self, debug_mode)
# Add the configs through extend() for validation
self.configs = Util.Container()
self.extend(configs)
self.ident_re = re.compile(r"^([a-zA-Z_0-9]+)(.*)")
self.int_re = re.compile(r"^([0-9\-]+)(.*)")
self.op_re = re.compile(r"^(!=|==|=|!|<=|<|>=|>|&&|\|\|)(.*)")
self.whitespace_re = re.compile(r"^\s+(.*)")
self.implicit_configs = Util.Container()
self.expr = None
def __init__(self,
filename="",
err_ctx=Log.ERROR.RUNTIME,
err_fatal=False):
self.err_ctx = err_ctx
self.err_fatal = err_fatal
self.filename = filename
self.configs = Util.Container() # Initially empty
# Parallel containers for metadata associated with configs
self.comments = Util.Container() # 0-1:1 with configs
self.lines = Util.Container() # 1:1 with configs
self.settings = None
def test_complex_set(self):
prj = Project.new(err_fatal=True)
self.assertIsNone(prj.get_name())
prj.set_name("Complex Set")
self.assertEqual(prj.get_name(), "Complex Set")
prj.add_layer("layer1")
prj.add_variant("layer1", "variant1.1", "l1_v1_1.cfg")
prj.add_variant("layer1", "variant1.2", "l1_v1_2.cfg")
prj.add_layer("layer2")
prj.add_variant("layer2", "variant2.1", "l2_v2_1.cfg")
prj.add_package("path/to/pkg1.mfg")
prj.add_package("package2.mfg",
module_root=Project.ROOT.DEPENDENCY,
module_id="Package2")
self.assertEqual(
prj.get_target("layer1", "variant1.1").filename, "l1_v1_1.cfg")
self.assertEqual(
prj.get_target("layer1", "variant1.2").filename, "l1_v1_2.cfg")
self.assertEqual(
prj.get_target("layer2", "variant2.1").filename, "l2_v2_1.cfg")
prj.get_target("layer1", "variant1.1").config.cfg_a1 = "A1"
prj.get_target("layer1", "variant1.1").config.cfg_b1 = "B1"
prj.get_target("layer1", "variant1.2").config.cfg_a2 = "A2"
prj.get_target("layer2", "variant2.1").config.cfg_c1 = "C1"
self.verify_configs(
prj.get_target("layer1", "variant1.1").config,
Util.Container(cfg_a1="A1", cfg_b1="B1"))
self.verify_configs(
prj.get_target("layer1", "variant1.2").config,
Util.Container(cfg_a2="A2"))
self.verify_configs(
prj.get_target("layer2", "variant2.1").config,
Util.Container(cfg_c1="C1"))
self.assertEqual(prj.get_packages(), [
Util.Container(filename="path/to/pkg1.mfg",
file_root=Project.ROOT.SOURCE,
module_root=Project.ROOT.SOURCE,
module_id=None),
Util.Container(filename="package2.mfg",
file_root=Project.ROOT.SOURCE,
module_root=Project.ROOT.DEPENDENCY,
module_id="Package2")
])
def test_conditional_compact2(self):
for data in [
("1", ["a1","a2"]),
("2", ["b1","b2"]),
("3", ["c1","c2"])
]:
self.trace_msg = "sel = " + data[0]
self.create_parser(Util.Container(sel = data[0]))
self.parse_lines(
":sources",
":if(sel=1)",
" a1",
" a2",
":elif(sel=2)",
" b1",
" b2",
":else",
" c1",
" c2",
":end"
)
expected = create_empty_manifest_container()
expected.sources = data[1]
self.verify_manifest(expected)
def _config_end(self, context):
"""End a config block"""
abs_def_file = Util.get_abs_path(context.ctx.definition, self.pkg_root)
if self.validate_files or self.def_parser:
if not os.path.isfile(abs_def_file):
self.log_error("'{}' is not a file".format(abs_def_file))
cfg = Util.Container(definition=context.ctx.definition,
generators=context.ctx.generators)
if self.def_parser:
cfg["definition_abs"] = abs_def_file
cfg["def_tree"] = self.def_parser(abs_def_file)
self.settings.add_implicit_configs(
cfg.def_tree.get_implicit_values())
self.manifest.add_config(cfg)
def test_create_flag_enum(self):
testBitField = Util.create_flag_enum("a", "b", "c", "d")
self.assertEqual(testBitField.a, 0x1)
self.assertEqual(testBitField.b, 0x2)
self.assertEqual(testBitField.c, 0x4)
self.assertEqual(testBitField.d, 0x8)
self.assertEqual(0xf, testBitField.ALL)
def generate(self, definitions, out_dir):
"""Generate a settings file"""
java_file = self.filename.lower()
package_dir = os.path.dirname(java_file)
package_name = os.path.relpath(package_dir, start=out_dir)
package_name = self.PACKAGE_RE.sub(".", package_name)
class_name = self.CLASS_RE.sub("", os.path.basename(self.filename))
os.makedirs(Util.get_abs_path(package_dir, out_dir), exist_ok=True)
with LineReader.OpenFileCM(java_file, "wt") as hdr_cm:
if not hdr_cm:
Log.E("Could not open {}: {}".format(self.filename,
hdr_cm.get_err_msg()))
hdr_file = hdr_cm.get_file()
# File header
hdr_file.write("\n".join([
"/* GENERATED BY GLOBIFEST -- DO NOT EDIT */", "",
"package {}".format(package_name), "",
"public final class {}".format(class_name), "{\n"
]))
# Add values
template = " public final static {} {} = {};\n"
for d in definitions:
ptype = d.param.get_type()
pid = d.param.get_identifier()
value = d.value
# Write implicit values first
implicit_id = None
for implicit_value in d.param.get_implicit_values():
hdr_file.write(
template.format("int", implicit_value[0],
implicit_value[1]))
if value == implicit_value[1]:
implicit_id = implicit_value[0]
# Write the parameter
if ptype == DefTree.PARAM_TYPE.INT:
hdr_file.write(template.format("int", pid, value))
elif ptype == DefTree.PARAM_TYPE.FLOAT:
# Default type is double precision
hdr_file.write(template.format("double", pid, value))
elif ptype == DefTree.PARAM_TYPE.STRING:
hdr_file.write(template.format("String", pid, value))
elif ptype == DefTree.PARAM_TYPE.BOOL:
hdr_file.write(
template.format("boolean", pid, value.lower()))
elif ptype == DefTree.PARAM_TYPE.ENUM:
if implicit_id:
value = implicit_id
hdr_file.write(template.format("int", pid, value))
else:
# TODO: Handle more complex literal types
Log.E("Unhandled value of type {}".format(
str(d.param.ptype)))
# File footer
hdr_file.write("}\n")
def on_param(self, param):
"""Handle a parameter"""
try:
value = self.settings.get_value(param.get_identifier())
except KeyError:
Log.E("Undefined value {}".format(param))
self.out.append(Util.Container(param=param, value=value))
def test_compare_ints(self):
self.create_config_set(Util.Container(a="1", b="2", c="10"))
self.assertTrue(self.config.evaluate("a==1"))
self.assertFalse(self.config.evaluate("a!=1"))
self.assertFalse(self.config.evaluate("a<0"))
self.assertFalse(self.config.evaluate("a<1"))
self.assertTrue(self.config.evaluate("a<2"))
self.assertFalse(self.config.evaluate("a<=0"))
self.assertTrue(self.config.evaluate("a<=1"))
self.assertTrue(self.config.evaluate("a<=2"))
self.assertFalse(self.config.evaluate("a>2"))
self.assertFalse(self.config.evaluate("a>1"))
self.assertTrue(self.config.evaluate("a>0"))
self.assertFalse(self.config.evaluate("a>=2"))
self.assertTrue(self.config.evaluate("a>=1"))
self.assertTrue(self.config.evaluate("a>=0"))
# Ensure the numeric comparison works with two identifiers
self.assertTrue(self.config.evaluate("a < b"))
# Ensure the comparison is numeric, not string
self.assertTrue(self.config.evaluate("2 < 10"))
self.assertTrue(self.config.evaluate("b < c"))
def test_flat_manifest(self):
self.create_parser()
self.parse_lines(
":prv_includes",
" .",
":pub_includes",
" include",
" xyz/lib",
":pub_defines",
" __ABC_MODULE_VER__=100",
" __ABC_LIST__",
":prv_defines",
" __xyz_static__",
" __xyz_nodeprecate__",
":sources",
" abc_module.c",
" abc_list.c",
" xyz/xyz_lib.c",
":aux_files",
" README.md",
" LICENSE",
" xyz/LICENSE",
" xyz/README.md"
)
expected = Util.Container(
aux_files = ["README.md", "LICENSE", "xyz/LICENSE", "xyz/README.md"],
prv_defines = ["__xyz_static__", "__xyz_nodeprecate__"],
prv_includes = ["."],
pub_defines = ["__ABC_MODULE_VER__=100", "__ABC_LIST__"],
pub_includes = ["include", "xyz/lib"],
sources = ["abc_module.c", "abc_list.c", "xyz/xyz_lib.c"]
)
self.verify_manifest(expected)
def test_conditional_long1(self):
for data in [
("1", ["a"]),
("2", ["b"]),
("3", ["c"]),
("4", ["d"])
]:
self.trace_msg = "sel = " + data[0]
self.create_parser(Util.Container(sel = data[0]))
self.parse_lines(
":sources",
":if",
" (",
" sel=1",
" )",
" a",
":elif",
" (",
" sel=2",
" )",
" b",
":elif",
" (",
" sel==3",
" )",
" c",
":else",
" d",
":end"
)
expected = create_empty_manifest_container()
expected.sources = data[1]
self.verify_manifest(expected)
class Context(object):
"""
Base class for parsing contexts
"""
CTYPE = Util.create_enum("CONDITION", "CONFIG")
def __init__(self, manifest_parser, prev_context, line_info, ctype):
"""Initializes shared context info"""
self.manifest_parser = manifest_parser
self.ctype = ctype
if prev_context is None:
self.level = 0
self.line_info = "<invalid>"
else:
# Derive context from previous
self.level = prev_context.level + 1
self.line_info = line_info
# Link to the previous context
self.prev_context = prev_context
def get_ctype(self):
"""Returns the context type"""
return self.ctype
def has_parameters(self):
"""Returns whether this type of context has parameters"""
if self.ctype == Context.CTYPE.CONDITION:
return False
return True
def test_access_values(self):
self.create_config_set(Util.Container(a="1", b="2", c="3"))
# Test presence and read value accessors
self.assertTrue(self.config.has_value("a"))
self.assertEqual(self.config.get_value("a"), "1")
self.assertTrue(self.config.has_value("b"))
self.assertEqual(self.config.get_value("b"), "2")
self.assertTrue(self.config.has_value("c"))
self.assertEqual(self.config.get_value("c"), "3")
self.assertFalse(self.config.has_value("d"))
# Test undefining an existing value
self.config.undefine("b")
self.assertFalse(self.config.has_value("b"))
# Test undefining a nonexistent value
self.config.undefine("d")
self.assertFalse(self.config.has_value("d"))
# Test setting an existing value
self.config.set_value("a", "0")
self.assertTrue(self.config.has_value("a"))
self.assertEqual(self.config.get_value("a"), "0")
# Test setting a nonexistent value
self.config.set_value("d", "4")
self.assertTrue(self.config.has_value("d"))
self.assertEqual(self.config.get_value("d"), "4")
def _verify_scope(self, scope, expected, name=None):
# verify the name
self.cur_scope.append(name or scope.get_name())
# Verify the description
expected_desc = expected.get("description", "")
self.assertEqual(expected_desc, scope.get_description())
# Verify the parameters
expected_params = expected.get("params", [])
params = list(str(p) for p in scope.get_params())
self.assertListEqual(expected_params, params)
# Copy the list of expected children, so we can modify it as we traverse
no_children = Util.Container()
expected_children = copy.copy(expected.get("children", no_children))
# Match up actual children with expected children
for actual_child in scope.get_children():
expected_child = expected_children.get(actual_child[0], None)
if expected_child is None:
self.fail("Unexpected child {}".format(actual_child))
else:
# Matched a child by identifier, verify the sub-scope
self._verify_scope(actual_child[1], expected_child)
expected_children.pop(actual_child[0])
if expected_children:
# Print out the missing children
self.fail("Missing children: {}".format(",".join(
expected_children.keys())))
self.cur_scope.pop()
def test_ident_as_value(self):
self.create_config_set(
Util.Container(bTrue="TRUE", bFalse="FALSE", i1="1", i0="0"))
self.assertTrue(self.config.evaluate("bTrue"))
self.assertFalse(self.config.evaluate("bFalse"))
self.assertTrue(self.config.evaluate("i1"))
self.assertFalse(self.config.evaluate("i0"))
def __init__(self, debug_mode=False, init_state=0):
"""
Initialize the class
"""
Log.Debuggable.__init__(self, debug_mode)
self._sm_base = Util.Container(state=init_state, new_state=init_state)
self.title = "state"
def test_create_enum(self):
testEnum = Util.create_enum("a", "b", "c", "d")
self.assertEqual(testEnum.a, 0)
self.assertEqual(testEnum.b, 1)
self.assertEqual(testEnum.c, 2)
self.assertEqual(testEnum.d, 3)
self.assertEqual(testEnum.COUNT, 4)
self.assertEqual(testEnum.enum_id, ("a", "b", "c", "d"))
def read_project(in_fname, out_dir):
"""
Read project
@return Tuple containing (Project object with parsed result,
Project directory, output directory)
"""
project = Project.new(in_fname, err_fatal=True)
parser = ProjectParser.new(project)
reader = LineReader.new(parser)
reader.read_file_by_name(in_fname)
cwd = os.getcwd()
prj_dir = Util.get_abs_path(os.path.dirname(project.get_filename()), cwd)
out_dir = Util.get_abs_path(out_dir, cwd)
return (project, prj_dir, out_dir)
def test_enum2(self):
enum_param = DefTree.Parameter(
pid="IDENT_ENUM_TYPE",
ptitle="Sample Enum",
ptype=DefTree.PARAM_TYPE.ENUM,
# Empty description
# No explicit default
metadata=Util.Container(vlist=[
Util.Container(id="IDENT_ENUM_1", text="\"Text 1\""),
Util.Container(id="IDENT_ENUM_2", text="\"Text 2\"")
]))
self.assertEqual(
str(enum_param), "id=IDENT_ENUM_TYPE type=ENUM title=Sample Enum" +
" choice=IDENT_ENUM_1,\"Text 1\"" +
" choice=IDENT_ENUM_2,\"Text 2\"")
self.assertListEqual(enum_param.get_implicit_values(),
[("IDENT_ENUM_1", "0"), ("IDENT_ENUM_2", "1")])
def __init__(self, fname, root=None):
self.out = Util.Container()
self.fname = fname
self.configs = []
if root is None:
self.root = os.path.dirname(self.fname)
else:
self.root = root
def test_ident_as_ident(self):
self.create_config_set(
Util.Container(bActualValue="TRUE",
bIdentValue="bActualValue",
iActualValue="1",
iIdentValue="iActualValue"))
self.assertTrue(self.config.evaluate("bIdentValue"))
self.assertTrue(self.config.evaluate("iIdentValue == 1"))
def create_parser(self, configs = Util.Container()):
# The manifest and reader are not under test, but simple enough to use directly
self.manifest = Helpers.new_manifest()
self.configs = Helpers.new_settings(configs)
self.parser = ManifestParser.new(self.manifest, self.configs, debug_mode=True, validate_files=False)
# The reader is not under test, but it provides a good way to feed strings to the parser
self.reader = LineReader.new(self.parser)
def test_quick_values(self):
self.create_parser()
self.parse_lines(":config_b MY_BOOL_CONFIG",
":config_s MY_STRING_CONFIG",
":config_i MY_INT_CONFIG",
":config_f MY_FLOAT_CONFIG")
self.verify_deftree(
Util.Container(
**{
"/":
Util.Container(params=[
"id=MY_BOOL_CONFIG type=BOOL",
"id=MY_STRING_CONFIG type=STRING",
"id=MY_INT_CONFIG type=INT",
"id=MY_FLOAT_CONFIG type=FLOAT"
])
}))
def test_bool(self):
self.create_config_set()
self.assertTrue(self.config.evaluate("TRUE"))
self.assertTrue(self.config.evaluate("TRUE"))
self.create_config_set(Util.Container(v="TRUE"))
self.assertTrue(self.config.evaluate("v == TRUE"))
self.assertTrue(self.config.evaluate("v && TRUE"))
self.assertFalse(self.config.evaluate("!v"))
