def test_pure_policies(self):
xproto = """
policy my_policy < exists x:a=b >
"""
proto = """
option my_policy = "policy:< exists x:a=b >";
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{{ policies }}
"""
)
args_xproto = XOSProcessorArgs()
args_xproto.inputs = xproto
args_xproto.target = target
xproto_gen = XOSProcessor.process(args_xproto)
args_proto = XOSProcessorArgs()
args_proto.inputs = proto
args_proto.target = target
args_proto.rev = True
proto_gen = XOSProcessor.process(args_proto)
self.assertEqual(proto_gen, xproto_gen)
def generate_service_models(service_dir, service_dest_dir, service_name):
"""
Generate the django code starting from xProto for a given service.
:param service_dir: string (path to the folder)
:param service_name: string (name of the service)
:return: void
"""
sync_dir = os.path.join(service_dir, "xos/synchronizer/models")
xprotos = find_xproto_in_folder(sync_dir)
decls = find_decls_models(sync_dir)
log.debug("Generating models for %s from files %s" % (service_name, ", ".join(xprotos)))
out_dir = os.path.join(service_dest_dir, service_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
args = XOSProcessorArgs(
output=out_dir,
files=xprotos,
target="service.xtarget",
write_to_file="target",
)
XOSProcessor.process(args)
security_args = XOSProcessorArgs(
output=out_dir,
target="django-security.xtarget",
dest_file="security.py",
write_to_file="single",
files=xprotos,
)
XOSProcessor.process(security_args)
init_py_filename = os.path.join(out_dir, "__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# copy over models.py files from the service
if len(decls) > 0:
for file in decls:
fn = os.path.basename(file)
src_fn = file
dest_fn = os.path.join(out_dir, fn)
log.debug("Copying models.py from %s to %s" % (src_fn, dest_fn))
shutil.copyfile(src_fn, dest_fn)
# copy existing migrations from the service, otherwise they won't be incremental
src_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
if os.path.isdir(src_dir):
dest_dir = os.path.join(out_dir, "migrations")
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
shutil.copytree(src_dir, dest_dir)
def test_field_numbers(self):
args = XOSProcessorArgs(
files=[REVERSEFIELDTEST_XPROTO], target=FIELDTEST_TARGET
)
output = XOSProcessor.process(args)
def _assert_field(modelname, fieldname, id):
self.assertIn("%s,%s,%s" % (modelname, fieldname, id), output)
# rel_int1s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1as_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1a", "rel_int1s_ids", 1001)
_assert_field("Leaf1a", "rel_leaf1as_ids", 1101)
# rel_int2s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1bs_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1b", "rel_int1s_ids", 1001)
_assert_field("Leaf1b", "rel_leaf1bs_ids", 1101)
# There are no reverse numbers specified for Intermediate2 or Leaf2, so xproto will fall back to automatic
# numbering starting at 1900.
_assert_field("Leaf2", "rel_int2s_ids", 1900)
_assert_field("Leaf2", "rel_leaf2s_ids", 1901)
def test_equal(self):
xproto = """
policy output < not (ctx.user = obj.user) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.user == obj.user)
i1 = (not i2)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
policy_output_validator(obj, ctx)
def test_django_with_attic(self):
"""
[XOS-GenX] Generate django output from test.xproto
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target="django.xtarget",
attic=TEST_ATTICS,
output=OUTPUT_DIR,
dest_extension="py",
write_to_file="model",
)
output = XOSProcessor.process(args)
# xosmodel has custom header attic
self.assertIn("from core.models.xosbase import *", output["XOSModel"])
self.assertIn("class XOSModel_decl(XOSBase):", output["XOSModel"])
# vrouter port use the default header
self.assertIn("from core.models.xosbase import *", output["VRouterPort"])
self.assertIn("class VRouterPort_decl(XOSBase):", output["VRouterPort"])
# verify files
xosmodel = OUTPUT_DIR + "/xosmodel.py"
self.assertTrue(os.path.isfile(xosmodel))
xmf = open(xosmodel).read()
self.assertIn("from core.models.xosbase import *", xmf)
self.assertIn("class XOSModel_decl(XOSBase):", xmf)
vrouterport = OUTPUT_DIR + "/vrouterport.py"
self.assertTrue(os.path.isfile(vrouterport))
vrpf = open(vrouterport).read()
self.assertIn("from core.models.xosbase import *", vrpf)
self.assertIn("class VRouterPort_decl(XOSBase):", vrpf)
def _test_field_graph(self):
xproto = """
message VRouterDevice (PlCoreBase){
optional string name = 1 [help_text = "device friendly name", max_length = 20, null = True, db_index = False, blank = True, unique_with="openflow_id"];
required string openflow_id = 2 [help_text = "device identifier in ONOS", max_length = 20, null = False, db_index = False, blank = False, unique_with="name"];
required string config_key = 3 [default = "basic", max_length = 32, blank = False, help_text = "configuration key", null = False, db_index = False, unique_with="driver"];
required string driver = 4 [help_text = "driver type", max_length = 32, null = False, db_index = False, blank = False, unique_with="vrouter_service"];
required manytoone vrouter_service->VRouterService:devices = 5 [db_index = True, null = False, blank = False];
required string A = 6 [unique_with="B"];
required string B = 7 [unique_with="C"];
required string C = 8 [unique_with="A"];
required string D = 9;
required string E = 10 [unique_with="F,G"];
required string F = 11;
required string G = 12;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{{ xproto_field_graph_components(proto.messages.0.fields, proto.messages.0) }}
"""
)
args = XOSProcessorArgs(inputs=xproto, target=target)
output = XOSProcessor.process(args)
output = eval(output)
self.assertIn({"A", "B", "C"}, output)
self.assertIn({"openflow_id", "name"}, output)
self.assertIn({"config_key", "vrouter_service", "driver"}, output)
self.assertIn({"E", "F", "G"}, output)
union = reduce(lambda acc, x: acc | x, output)
self.assertNotIn("D", union)
def test_equal(self):
xproto = \
"""
policy output < ctx.user = obj.user >
"""
args = FakeArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
"""
obj = FakeArgs()
obj.user = 1
ctx = FakeArgs()
ctx.user = 1
verdict = output_security_check(obj, ctx)
def test_singularize(self):
proto = """
message TestSingularize {
// The following field has an explicitly specified singular
required int many = 1 [singular = "one"];
// The following fields have automatically computed singulars
required int sheep = 2;
required int slices = 2;
required int networks = 2;
required int omf_friendlies = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_singularize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"one,sheep,slice,network,omf_friendly", output.lstrip().rstrip().rstrip(",")
)
def test_pluralize(self):
proto = """
message TestPluralize {
// The following field has an explicitly specified plural
required int anecdote = 1 [plural = "data"];
// The following fields have automatically computed plurals
required int sheep = 2;
required int slice = 2;
required int network = 2;
required int omf_friendly = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_pluralize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"data,sheep,slices,networks,omf_friendlies",
output.lstrip().rstrip().rstrip(","),
)
def test_one_to_many_in_modeldef(self):
xproto = """
option app_label = "test";
message ServiceDependency {
required manytoone provider_service->Service:provided_dependencies = 1;
required manytoone subscriber_service->Service:subscribed_dependencies = 2;
}
message Service {
required string name = 1;
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
# Service deps model
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}", output
)
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}", output
)
# Service model
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
def test_basic_proto(self):
xtarget = XProtoTestHelpers.write_tmp_target("{{ proto }}")
xproto = """
message Person {
required string name = 1;
required int32 id = 2; // Unique ID number for this person.
optional string email = 3 [symphony = "da da da dum"];
enum PhoneType {
MOBILE = 0;
HOME = 1;
WORK = 2;
}
required string number = 1;
optional PhoneType type = 2;
repeated PhoneNumber phones = 4;
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = xtarget
output = XOSProcessor.process(args)
self.assertIn("PhoneNumber", output)
def test_django_with_attic(self):
"""
[XOS-GenX] Generate django output from test.xproto
"""
args = FakeArgs()
args.files = [TEST_XPROTO, VROUTER_XPROTO]
args.target = 'django.xtarget'
args.attic = TEST_ATTICS
args.output = OUTPUT_DIR
args.dest_extension = 'py'
args.write_to_file = 'model'
output = XOSProcessor.process(args)
# xosmodel has custom header attic
self.assertIn('from xosmodel_header import *', output['XOSModel'])
self.assertIn('class XOSModel(XOSBase):', output['XOSModel'])
# vrouter port use the default header
self.assertIn('header import *', output['VRouterPort'])
self.assertIn('class VRouterPort(XOSBase):', output['VRouterPort'])
#verify files
xosmodel = OUTPUT_DIR + '/xosmodel.py'
self.assertTrue(os.path.isfile(xosmodel))
xmf = open(xosmodel).read()
self.assertIn('from xosmodel_header import *', xmf)
self.assertIn('class XOSModel(XOSBase):', xmf)
vrouterport = OUTPUT_DIR + '/vrouterport.py'
self.assertTrue(os.path.isfile(vrouterport))
vrpf = open(vrouterport).read()
self.assertIn('header import *', vrpf)
self.assertIn('class VRouterPort(XOSBase):', vrpf)
def test_user_policy(self):
xproto = """
policy test_policy <
ctx.user.is_admin
| ctx.user.id = obj.id
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.permission = "role:admin"
& Privilege.object_type = "Site"
& Privilege.object_id = ctx.user.site.id) >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def policy_output_enforcer(obj, ctx):
i2 = ctx.user.is_admin
i4 = (ctx.user.id == obj.id)
i5 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(permission='role:admin'), Q(object_type='Site'), Q(object_id=ctx.user.site.id))[0]
i3 = (i4 or i5)
i1 = (i2 or i3)
return i1
"""
# FIXME: Test this policy by executing it
self.assertTrue(policy_output_enforcer is not None)
def test_slice_name_validation(self):
xproto = """
policy test_policy < not obj.id -> {{ obj.name.startswith(obj.site.login_base) }} >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i3 = obj.id
i4 = obj.name.startswith(obj.site.login_base)
i2 = ((not i3) or i4)
i1 = (not i2)
if (not i1):
raise ValidationError('Necessary Failure')
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_instance_container(self):
xproto = """
policy test_policy < (obj.isolation = "container" | obj.isolation = "container_vm" ) -> (obj.image.kind = "container") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i4 = (obj.isolation == 'container')
i5 = (self.isolation == 'container_vm')
i2 = (i4 or i5)
i3 = (obj.image.kind == 'container')
i1 = (i2 or i3)
return i1
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals(
)) # This loads the generated function, which should look like this:
"""
def policy_sub_policy_validator(obj, ctx):
i1 = (ctx.user == obj.user)
if (not i1):
raise ValidationError('Necessary Failure')
def policy_output_validator(obj, ctx):
i1 = policy_sub_policy_validator(obj.child, ctx)
if (not i1):
raise ValidationError('Necessary Failure')
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
verdict = policy_output_enforcer(obj, ctx)
def test_instance_container(self):
xproto = """
policy test_policy < (obj.isolation = "container" | obj.isolation = "container_vm" ) -> (obj.image.kind = "container") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i4 = (obj.isolation == 'container')
i5 = (self.isolation == 'container_vm')
i2 = (i4 or i5)
i3 = (obj.image.kind == 'container')
i1 = (i2 or i3)
return i1
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_optional_relations(self):
"""
[XOS-GenX] Generate DJANGO models, verify relations
"""
xproto = """
option app_label = "test";
message ENodeB {
}
message Handover {
}
message Foo {
optional manytoone enodeb->ENodeB:profiles = 1 [null = True, blank = True];
required manytoone handover->Handover:profiles = 2 [null = False, blank = False];
}
"""
args = XOSProcessorArgs(inputs=xproto, target="django.xtarget")
output = XOSProcessor.process(args)
null_true = [s for s in output.splitlines() if "null = True" in s]
null_false = [s for s in output.splitlines() if "null = False" in s]
blank_true = [s for s in output.splitlines() if "blank = True" in s]
blank_false = [s for s in output.splitlines() if "blank = False" in s]
self.assertEqual(len(null_true), 1)
self.assertEqual(len(null_false), 1)
self.assertEqual(len(blank_true), 1)
self.assertEqual(len(blank_false), 1)
def test_generator_custom_target_from_file(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(files=[TEST_XPROTO], target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_equal(self):
xproto = """
policy output < not (ctx.user = obj.user) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.user == obj.user)
i1 = (not i2)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
policy_output_validator(obj, ctx)
def test_django_with_attic(self):
"""
[XOS-GenX] Generate django output from test.xproto
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target="django.xtarget",
attic=TEST_ATTICS,
output=OUTPUT_DIR,
dest_extension="py",
write_to_file="model",
)
output = XOSProcessor.process(args)
# xosmodel has custom header attic
self.assertIn("from xosmodel_header import *", output["XOSModel"])
self.assertIn("class XOSModel(XOSBase):", output["XOSModel"])
# vrouter port use the default header
self.assertIn("header import *", output["VRouterPort"])
self.assertIn("class VRouterPort(XOSBase):", output["VRouterPort"])
# verify files
xosmodel = OUTPUT_DIR + "/xosmodel.py"
self.assertTrue(os.path.isfile(xosmodel))
xmf = open(xosmodel).read()
self.assertIn("from xosmodel_header import *", xmf)
self.assertIn("class XOSModel(XOSBase):", xmf)
vrouterport = OUTPUT_DIR + "/vrouterport.py"
self.assertTrue(os.path.isfile(vrouterport))
vrpf = open(vrouterport).read()
self.assertIn("header import *", vrpf)
self.assertIn("class VRouterPort(XOSBase):", vrpf)
def _test_field_graph(self):
xproto = """
message VRouterDevice (PlCoreBase){
optional string name = 1 [help_text = "device friendly name", max_length = 20, null = True, db_index = False, blank = True, unique_with="openflow_id"];
required string openflow_id = 2 [help_text = "device identifier in ONOS", max_length = 20, null = False, db_index = False, blank = False, unique_with="name"];
required string config_key = 3 [default = "basic", max_length = 32, blank = False, help_text = "configuration key", null = False, db_index = False, unique_with="driver"];
required string driver = 4 [help_text = "driver type", max_length = 32, null = False, db_index = False, blank = False, unique_with="vrouter_service"];
required manytoone vrouter_service->VRouterService:devices = 5 [db_index = True, null = False, blank = False];
required string A = 6 [unique_with="B"];
required string B = 7 [unique_with="C"];
required string C = 8 [unique_with="A"];
required string D = 9;
required string E = 10 [unique_with="F,G"];
required string F = 11;
required string G = 12;
}
"""
target = XProtoTestHelpers.write_tmp_target("""
{{ xproto_field_graph_components(proto.messages.0.fields, proto.messages.0) }}
""")
args = XOSProcessorArgs(inputs=xproto, target=target)
output = XOSProcessor.process(args)
output = eval(output)
self.assertIn({"A", "B", "C"}, output)
self.assertIn({"openflow_id", "name"}, output)
self.assertIn({"config_key", "vrouter_service", "driver"}, output)
self.assertIn({"E", "F", "G"}, output)
union = reduce(lambda acc, x: acc | x, output)
self.assertNotIn("D", union)
def test_controller_network_policy(self):
xproto = """
policy test_policy <
ctx.user.is_admin
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Slice"
& Privilege.object_id = obj.owner.id)
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Site"
& Privilege.object_id = obj.owner.site.id
& Privilege.permission = "role:admin") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_enforcer(obj, ctx):
i2 = ctx.user.is_admin
i4 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Slice'), Q(object_id=obj.owner.id))[0]
i5 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Site'), Q(object_id=obj.owner.site.id), Q(permission='role:admin'))[0]
i3 = (i4 or i5)
i1 = (i2 or i3)
return i1
"""
# FIXME: Test this policy by executing it
self.assertTrue(policy_output_enforcer is not None)
def test_bin(self):
xproto = """
policy output < ctx.is_admin = True | obj.empty = True>
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
return i1
"""
obj = FakeObject()
obj.empty = True
ctx = FakeObject()
ctx.is_admin = True
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_package_fqn(self):
args = XOSProcessorArgs()
target = XProtoTestHelpers.write_tmp_target("""
{% for m in proto.messages %}
{{ m.name }},{{ m.package }},{{ m.fqn }}
{% endfor %}
""")
xproto = """
package xos.core;
message Port (PlCoreBase,ParameterMixin) {
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Port,xos.core,xos.core.Port", output)
def test_basic_proto(self):
xtarget = XProtoTestHelpers.write_tmp_target("{{ proto }}")
xproto = \
"""
message Person {
required string name = 1;
required int32 id = 2; // Unique ID number for this person.
optional string email = 3 [symphony = "da da da dum"];
enum PhoneType {
MOBILE = 0;
HOME = 1;
WORK = 2;
}
required string number = 1;
optional PhoneType type = 2;
repeated PhoneNumber phones = 4;
}
"""
args = FakeArgs()
args.inputs = xproto
args.target = xtarget
output = XOSProcessor.process(args)
self.assertIn("PhoneNumber", output)
def test_package_fqn(self):
args = XOSProcessorArgs()
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages %}
{{ m.name }},{{ m.package }},{{ m.fqn }}
{% endfor %}
"""
)
xproto = """
package xos.core;
message Port (PlCoreBase,ParameterMixin) {
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Port,xos.core,xos.core.Port", output)
def test_singularize(self):
proto = """
message TestSingularize {
// The following field has an explicitly specified singular
required int many = 1 [singular = "one"];
// The following fields have automatically computed singulars
required int sheep = 2;
required int slices = 2;
required int networks = 2;
required int omf_friendlies = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_singularize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"one,sheep,slice,network,omf_friendly", output.lstrip().rstrip().rstrip(",")
)
def test_pluralize(self):
proto = """
message TestPluralize {
// The following field has an explicitly specified plural
required int anecdote = 1 [plural = "data"];
// The following fields have automatically computed plurals
required int sheep = 2;
required int slice = 2;
required int network = 2;
required int omf_friendly = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_pluralize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"data,sheep,slices,networks,omf_friendlies",
output.lstrip().rstrip().rstrip(","),
)
def test_generator_custom_target_from_inputs(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(inputs=open(TEST_XPROTO).read(), target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(
output, globals()
) # This loads the generated function, which should look like this:
"""
def policy_sub_policy_validator(obj, ctx):
i1 = (ctx.user == obj.user)
if (not i1):
raise ValidationError('Necessary Failure')
def policy_output_validator(obj, ctx):
i1 = policy_sub_policy_validator(obj.child, ctx)
if (not i1):
raise ValidationError('Necessary Failure')
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
verdict = policy_output_enforcer(obj, ctx)
def test_bin(self):
xproto = """
policy output < (ctx.is_admin = True | obj.empty = True) | False>
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.empty = False
ctx = FakeObject()
ctx.is_admin = False
with self.assertRaises(Exception):
verdict = policy_output_validator(obj, ctx)
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals(
)) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
def output_security_check(obj, ctx):
if obj.child:
i1 = sub_policy_security_check(obj.child, ctx)
else:
i1 = True
return i1
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
def output_security_check(obj, ctx):
if obj.child:
i1 = sub_policy_security_check(obj.child, ctx)
else:
i1 = True
return i1
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_bin(self):
xproto = """
policy output < (ctx.is_admin = True | obj.empty = True) | False>
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.empty = False
ctx = FakeObject()
ctx.is_admin = False
with self.assertRaises(Exception):
verdict = policy_output_validator(obj, ctx)
def test_one_to_many_in_modeldef(self):
xproto = """
option app_label = "test";
message ServiceDependency {
required manytoone provider_service->Service:provided_dependencies = 1;
required manytoone subscriber_service->Service:subscribed_dependencies = 2;
}
message Service {
required string name = 1;
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
# Service deps model
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}",
output)
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}",
output)
# Service model
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
def test_unfiltered(self):
""" With no include_* args, should get all models """
args = XOSProcessorArgs(files=[FILTERTEST_XPROTO],
target=FILTERTEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model2,Model3,")
def test_slice_name_validation(self):
xproto = """
policy test_policy < not obj.id -> {{ obj.name.startswith(obj.site.login_base) }} >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i3 = obj.id
i4 = obj.name.startswith(obj.site.login_base)
i2 = ((not i3) or i4)
i1 = (not i2)
if (not i1):
raise ValidationError('Necessary Failure')
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_max_length_zero(self):
args = XOSProcessorArgs()
args.files = ["/tmp/testvalidator.xproto"]
open("/tmp/testvalidator.xproto", "w").write("""
option app_label = "test";
message Port (XOSBase){
required string foo = 1 [max_length=0];
}
""")
args.target = "modeldefs.xtarget"
with patch.object(XProtoValidator, "print_errors",
autospec=True) as print_errors:
print_errors.return_value = None
output = XOSProcessor.process(args)
self.assertEqual(print_errors.call_count, 1)
validator = print_errors.call_args[0][0]
self.assertEqual(len(validator.errors), 1)
self.assertEqual(validator.errors[0]["severity"], "ERROR")
self.assertEqual(validator.errors[0]["message"],
"max_length should not be zero")
def test_bin(self):
xproto = """
policy output < ctx.is_admin = True | obj.empty = True>
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals(
)) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
return i1
"""
obj = FakeObject()
obj.empty = True
ctx = FakeObject()
ctx.is_admin = True
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_generator_custom_target_from_file(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(files=[TEST_XPROTO], target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_generator_custom_target_from_inputs(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(inputs=open(TEST_XPROTO).read(),
target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_filter_apps(self):
""" Should only get models whose apps are specified by include_apps """
args = XOSProcessorArgs(files=[FILTERTEST_XPROTO],
target=FILTERTEST_TARGET,
include_apps=["core"])
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model2,")
def test_filter_apps(self):
""" Should only get models whose apps are specified by include_apps """
args = XOSProcessorArgs(
files=[FILTERTEST_XPROTO], target=FILTERTEST_TARGET, include_apps=["core"]
)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model2,")
def test_base_class_fields(self):
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages %}
{{ m.name }} {
{%- for b in m.bases %}
{%- if proto.message_table[b.fqn] -%}
{%- set model = proto.message_table[b.fqn] %}
{% for f in model.fields %}
{{ f.type }} {{ f.name }};
{% endfor %}
{%- endif -%}
{% endfor %}
}
{% endfor %}
"""
)
xproto = """
package xos.network;
message Port (PlCoreBase,ParameterMixin){
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
package xos.someotherpackage;
message Instance (xos.network.Port){
optional string instance_id = 1 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance id", null = True, db_index = False];
optional string instance_uuid = 2 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance uuid", null = True, db_index = False];
required string name = 3 [max_length = 200, content_type = "stripped", blank = False, help_text = "Instance name", null = False, db_index = False];
optional string instance_name = 4 [max_length = 200, content_type = "stripped", blank = True, help_text = "OpenStack generated name", null = True, db_index = False];
optional string ip = 5 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
required manytoone image->Image:instances = 6 [db_index = True, null = False, blank = False];
optional manytoone creator->User:instances = 7 [db_index = True, null = True, blank = True];
required manytoone slice->Slice:instances = 8 [db_index = True, null = False, blank = False];
required manytoone deployment->Deployment:instance_deployment = 9 [db_index = True, null = False, blank = False];
required manytoone node->Node:instances = 10 [db_index = True, null = False, blank = False];
required int32 numberCores = 11 [help_text = "Number of cores for instance", default = 0, null = False, db_index = False, blank = False];
required manytoone flavor->Flavor:instance = 12 [help_text = "Flavor of this instance", default = "get_default_flavor()", null = False, db_index = True, blank = False];
optional string userData = 13 [help_text = "user_data passed to instance during creation", null = True, db_index = False, blank = True];
required string isolation = 14 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
optional string volumes = 15 [help_text = "Comma-separated list of directories to expose to parent context", null = True, db_index = False, blank = True];
optional manytoone parent->Instance:instance = 16 [help_text = "Parent Instance for containers nested inside of VMs", null = True, db_index = True, blank = True];
required manytomany tags->Tag = 17 [db_index = False, null = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("xos_created", output)
def test_base_class_fields(self):
target = XProtoTestHelpers.write_tmp_target("""
{% for m in proto.messages %}
{{ m.name }} {
{%- for b in m.bases %}
{%- if proto.message_table[b.fqn] -%}
{%- set model = proto.message_table[b.fqn] %}
{% for f in model.fields %}
{{ f.type }} {{ f.name }};
{% endfor %}
{%- endif -%}
{% endfor %}
}
{% endfor %}
""")
xproto =\
"""
package xos.network;
message Port (PlCoreBase,ParameterMixin){
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
package xos.someotherpackage;
message Instance (xos.network.Port){
optional string instance_id = 1 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance id", null = True, db_index = False];
optional string instance_uuid = 2 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance uuid", null = True, db_index = False];
required string name = 3 [max_length = 200, content_type = "stripped", blank = False, help_text = "Instance name", null = False, db_index = False];
optional string instance_name = 4 [max_length = 200, content_type = "stripped", blank = True, help_text = "OpenStack generated name", null = True, db_index = False];
optional string ip = 5 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
required manytoone image->Image:instances = 6 [db_index = True, null = False, blank = False];
optional manytoone creator->User:instances = 7 [db_index = True, null = True, blank = True];
required manytoone slice->Slice:instances = 8 [db_index = True, null = False, blank = False];
required manytoone deployment->Deployment:instance_deployment = 9 [db_index = True, null = False, blank = False];
required manytoone node->Node:instances = 10 [db_index = True, null = False, blank = False];
required int32 numberCores = 11 [help_text = "Number of cores for instance", default = 0, null = False, db_index = False, blank = False];
required manytoone flavor->Flavor:instance = 12 [help_text = "Flavor of this instance", default = "get_default_flavor()", null = False, db_index = True, blank = False];
optional string userData = 13 [help_text = "user_data passed to instance during creation", null = True, db_index = False, blank = True];
required string isolation = 14 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
optional string volumes = 15 [help_text = "Comma-separated list of directories to expose to parent context", null = True, db_index = False, blank = True];
optional manytoone parent->Instance:instance = 16 [help_text = "Parent Instance for containers nested inside of VMs", null = True, db_index = True, blank = True];
required manytomany tags->Tag = 17 [db_index = False, null = False, blank = True];
}
"""
args = FakeArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn('xos_created', output)
def test_xproto_lib(self):
target = XProtoTestHelpers.write_tmp_target("""
{{ xproto_first_non_empty([None, None, None, None, None, None, "Eureka"]) }}
""")
args = FakeArgs()
args.inputs = ''
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Eureka", output)
def test_xproto_model_to_oneof_key(self):
"""
[XOS-GenX] in some models we need to have a combine key on variable fields, for example, keys can be subscriber_service_id + oneof(provider_service_id, provider_network_id)
"""
xproto = \
"""
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required string key_1 = 2 [ null = "False", blank="False", tosca_key_one_of = "key_2"];
required string key_2 = 3 [ null = "False", blank="False", tosca_key_one_of = "key_1"];
required string key_3 = 4 [ null = "False", blank="False", tosca_key_one_of = "key_4"];
required string key_4 = 5 [ null = "False", blank="False", tosca_key_one_of = "key_3"];
}
"""
args = FakeArgs()
args.inputs = xproto
args.target = self.target_tosca_keys
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_4', 'key_3'], ['key_1', 'key_2']]", output)
xproto = \
"""
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required manytoone key_1->Bar:key_1s = 2;
required manytoone key_2->Bar:key_2s = 3 [tosca_key_one_of = "key_1"];
required manytoone key_3->Bar:key_3s = 4 [tosca_key_one_of = "key_1"];
}
"""
args.inputs = xproto
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_1_id', 'key_3_id', 'key_2_id']]", output)
def generate_swagger_docs(xproto):
# if not os.path.isfile(xproto):
# print "ERROR: Couldn't find xproto file for %s at: %s" % (service, xproto)
# return
print("Generating swagger docs for %s" % (xproto))
args = XOSProcessorArgs()
args.files = xproto
args.target = 'swagger.xtarget'
args.output = SWAGGER_DOCS_DIR
args.write_to_file = "single"
args.dest_file = "swagger.yaml"
args.quiet = False
try:
XOSProcessor.process(args)
except Exception:
print("ERROR: Couldn't generate swagger specs")
traceback.print_exc()
def test_context(self):
target = XProtoTestHelpers.write_tmp_target("""
{{ context.what }}
""")
args = FakeArgs()
args.inputs = ''
args.target = target
args.kv = 'what:what is what'
output = XOSProcessor.process(args)
self.assertIn("what is what", output)
def generate_swagger_docs(xproto):
# if not os.path.isfile(xproto):
# print "ERROR: Couldn't find xproto file for %s at: %s" % (service, xproto)
# return
print "Generating swagger docs for %s" % (xproto)
args = XOSProcessorArgs()
args.files = xproto
args.target = 'swagger.xtarget'
args.output = SWAGGER_DOCS_DIR
args.write_to_file = "single"
args.dest_file = "swagger.yaml"
args.quiet = False
try:
XOSProcessor.process(args)
except Exception:
print "ERROR: Couldn't generate swagger specs"
traceback.print_exc()
def test_filter_models(self):
""" Should only get models specified by include_models """
args = XOSProcessorArgs(
files=[FILTERTEST_XPROTO],
target=FILTERTEST_TARGET,
include_models=["Model1", "Model3"],
)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model3,")
def test_forall(self):
# This one we only parse
xproto = """
policy output < forall Credential: Credential.obj_id = obj_id >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals())
"""
def test_filter_models(self):
""" Should only get models specified by include_models """
args = XOSProcessorArgs(
files=[FILTERTEST_XPROTO],
target=FILTERTEST_TARGET,
include_models=["Model1", "Model3"],
)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model3,")
def test_xproto_model_to_oneof_key(self):
"""
[XOS-GenX] in some models we need to have a combine key on variable fields, for example, keys can be subscriber_service_id + oneof(provider_service_id, provider_network_id)
"""
xproto = """
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required string key_1 = 2 [ null = "False", blank="False", tosca_key_one_of = "key_2"];
required string key_2 = 3 [ null = "False", blank="False", tosca_key_one_of = "key_1"];
required string key_3 = 4 [ null = "False", blank="False", tosca_key_one_of = "key_4"];
required string key_4 = 5 [ null = "False", blank="False", tosca_key_one_of = "key_3"];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target_tosca_keys
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_1', 'key_2'], ['key_3', 'key_4']]", output)
xproto = """
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required manytoone key_1->Bar:key_1s = 2;
required manytoone key_2->Bar:key_2s = 3 [tosca_key_one_of = "key_1"];
required manytoone key_3->Bar:key_3s = 4 [tosca_key_one_of = "key_1"];
}
"""
args.inputs = xproto
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_1_id', 'key_2_id', 'key_3_id']]", output)
def test_xproto_lib(self):
target = XProtoTestHelpers.write_tmp_target(
"""
{{ xproto_first_non_empty([None, None, None, None, None, None, "Eureka"]) }}
"""
)
args = XOSProcessorArgs()
args.inputs = ""
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Eureka", output)
def test_write_multiple_files_from_xtarget(self):
"""
[XOS-GenX] read multiple models as input, print separate files based on +++
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target=SPLIT_TARGET,
output=OUTPUT_DIR,
write_to_file="target",
)
XOSProcessor.process(args)
generated_files = [f for f in os.listdir(OUTPUT_DIR) if not f.startswith(".")]
self.assertEqual(len(generated_files), 2)
xosmodel = open(os.path.join(OUTPUT_DIR, "xosmodel.txt"), "r").read()
vrouterport = open(os.path.join(OUTPUT_DIR, "vrouterport.txt"), "r").read()
self.assertIn("name: XOSModel", xosmodel)
self.assertIn("name: VRouterPort", vrouterport)
def test_forall(self):
# This one we only parse
xproto = """
policy output < forall Credential: Credential.obj_id = obj_id >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals())
"""
def test_proto_generator(self):
"""
[XOS-GenX] Generate DJANGO models, verify Fields and Foreign Keys
"""
args = XOSProcessorArgs(files=[VROUTER_XPROTO], target="django.xtarget")
output = XOSProcessor.process(args)
fields = [s for s in output.splitlines() if "Field(" in s]
self.assertEqual(len(fields), 2)
links = [s for s in output.splitlines() if "Key(" in s]
self.assertEqual(len(links), 2)
