def test_pure_policies(self):
xproto = """
policy my_policy < exists x:a=b >
"""
proto = """
option my_policy = "policy:< exists x:a=b >";
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{{ policies }}
"""
)
args_xproto = XOSProcessorArgs()
args_xproto.inputs = xproto
args_xproto.target = target
xproto_gen = XOSProcessor.process(args_xproto)
args_proto = XOSProcessorArgs()
args_proto.inputs = proto
args_proto.target = target
args_proto.rev = True
proto_gen = XOSProcessor.process(args_proto)
self.assertEqual(proto_gen, xproto_gen)
def generate_service_models(service_dir, service_dest_dir, service_name):
"""
Generate the django code starting from xProto for a given service.
:param service_dir: string (path to the folder)
:param service_name: string (name of the service)
:return: void
"""
sync_dir = os.path.join(service_dir, "xos/synchronizer/models")
xprotos = find_xproto_in_folder(sync_dir)
decls = find_decls_models(sync_dir)
log.debug("Generating models for %s from files %s" % (service_name, ", ".join(xprotos)))
out_dir = os.path.join(service_dest_dir, service_name)
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
args = XOSProcessorArgs(
output=out_dir,
files=xprotos,
target="service.xtarget",
write_to_file="target",
)
XOSProcessor.process(args)
security_args = XOSProcessorArgs(
output=out_dir,
target="django-security.xtarget",
dest_file="security.py",
write_to_file="single",
files=xprotos,
)
XOSProcessor.process(security_args)
init_py_filename = os.path.join(out_dir, "__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# copy over models.py files from the service
if len(decls) > 0:
for file in decls:
fn = os.path.basename(file)
src_fn = file
dest_fn = os.path.join(out_dir, fn)
log.debug("Copying models.py from %s to %s" % (src_fn, dest_fn))
shutil.copyfile(src_fn, dest_fn)
# copy existing migrations from the service, otherwise they won't be incremental
src_dir = os.path.join(service_dir, "xos", "synchronizer", "migrations")
if os.path.isdir(src_dir):
dest_dir = os.path.join(out_dir, "migrations")
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir) # empty the folder, we'll copy everything again
shutil.copytree(src_dir, dest_dir)
def test_basic_proto(self):
xtarget = XProtoTestHelpers.write_tmp_target("{{ proto }}")
xproto = """
message Person {
required string name = 1;
required int32 id = 2; // Unique ID number for this person.
optional string email = 3 [symphony = "da da da dum"];
enum PhoneType {
MOBILE = 0;
HOME = 1;
WORK = 2;
}
required string number = 1;
optional PhoneType type = 2;
repeated PhoneNumber phones = 4;
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = xtarget
output = XOSProcessor.process(args)
self.assertIn("PhoneNumber", output)
def test_equal(self):
xproto = """
policy output < not (ctx.user = obj.user) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.user == obj.user)
i1 = (not i2)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
policy_output_validator(obj, ctx)
def test_optional_relations(self):
"""
[XOS-GenX] Generate DJANGO models, verify relations
"""
xproto = """
option app_label = "test";
message ENodeB {
}
message Handover {
}
message Foo {
optional manytoone enodeb->ENodeB:profiles = 1 [null = True, blank = True];
required manytoone handover->Handover:profiles = 2 [null = False, blank = False];
}
"""
args = XOSProcessorArgs(inputs=xproto, target="django.xtarget")
output = XOSProcessor.process(args)
null_true = [s for s in output.splitlines() if "null = True" in s]
null_false = [s for s in output.splitlines() if "null = False" in s]
blank_true = [s for s in output.splitlines() if "blank = True" in s]
blank_false = [s for s in output.splitlines() if "blank = False" in s]
self.assertEqual(len(null_true), 1)
self.assertEqual(len(null_false), 1)
self.assertEqual(len(blank_true), 1)
self.assertEqual(len(blank_false), 1)
def test_slice_name_validation(self):
xproto = """
policy test_policy < not obj.id -> {{ obj.name.startswith(obj.site.login_base) }} >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i3 = obj.id
i4 = obj.name.startswith(obj.site.login_base)
i2 = ((not i3) or i4)
i1 = (not i2)
if (not i1):
raise ValidationError('Necessary Failure')
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_instance_container(self):
xproto = """
policy test_policy < (obj.isolation = "container" | obj.isolation = "container_vm" ) -> (obj.image.kind = "container") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
obj = FakeObject()
obj.isolation = "container"
obj.kind = "not a container"
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i4 = (obj.isolation == 'container')
i5 = (self.isolation == 'container_vm')
i2 = (i4 or i5)
i3 = (obj.image.kind == 'container')
i1 = (i2 or i3)
return i1
"""
with self.assertRaises(Exception):
policy_output_validator(obj, {})
def test_one_to_many_in_modeldef(self):
xproto = """
option app_label = "test";
message ServiceDependency {
required manytoone provider_service->Service:provided_dependencies = 1;
required manytoone subscriber_service->Service:subscribed_dependencies = 2;
}
message Service {
required string name = 1;
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = "modeldefs.xtarget"
output = XOSProcessor.process(args)
# Service deps model
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}", output
)
self.assertIn(
"{model: Service, type: manytoone, on_field: provider_service}", output
)
# Service model
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
self.assertIn(
"{model: ServiceDependency, type: onetomany, on_field: provider_service}",
output,
)
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(
output, globals()
) # This loads the generated function, which should look like this:
"""
def policy_sub_policy_validator(obj, ctx):
i1 = (ctx.user == obj.user)
if (not i1):
raise ValidationError('Necessary Failure')
def policy_output_validator(obj, ctx):
i1 = policy_sub_policy_validator(obj.child, ctx)
if (not i1):
raise ValidationError('Necessary Failure')
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
with self.assertRaises(Exception):
verdict = policy_output_enforcer(obj, ctx)
def test_call_policy(self):
xproto = """
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
def output_security_check(obj, ctx):
if obj.child:
i1 = sub_policy_security_check(obj.child, ctx)
else:
i1 = True
return i1
"""
obj = FakeObject()
obj.child = FakeObject()
obj.child.user = 1
ctx = FakeObject()
ctx.user = 1
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_field_numbers(self):
args = XOSProcessorArgs(
files=[REVERSEFIELDTEST_XPROTO], target=FIELDTEST_TARGET
)
output = XOSProcessor.process(args)
def _assert_field(modelname, fieldname, id):
self.assertIn("%s,%s,%s" % (modelname, fieldname, id), output)
# rel_int1s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1as_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1a", "rel_int1s_ids", 1001)
_assert_field("Leaf1a", "rel_leaf1as_ids", 1101)
# rel_int2s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1bs_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1b", "rel_int1s_ids", 1001)
_assert_field("Leaf1b", "rel_leaf1bs_ids", 1101)
# There are no reverse numbers specified for Intermediate2 or Leaf2, so xproto will fall back to automatic
# numbering starting at 1900.
_assert_field("Leaf2", "rel_int2s_ids", 1900)
_assert_field("Leaf2", "rel_leaf2s_ids", 1901)
def test_package_fqn(self):
args = XOSProcessorArgs()
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages %}
{{ m.name }},{{ m.package }},{{ m.fqn }}
{% endfor %}
"""
)
xproto = """
package xos.core;
message Port (PlCoreBase,ParameterMixin) {
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Port,xos.core,xos.core.Port", output)
def test_generator_custom_target_from_inputs(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(inputs=open(TEST_XPROTO).read(), target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_pluralize(self):
proto = """
message TestPluralize {
// The following field has an explicitly specified plural
required int anecdote = 1 [plural = "data"];
// The following fields have automatically computed plurals
required int sheep = 2;
required int slice = 2;
required int network = 2;
required int omf_friendly = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_pluralize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"data,sheep,slices,networks,omf_friendlies",
output.lstrip().rstrip().rstrip(","),
)
def test_generator_custom_target_from_file(self):
"""
[XOS-GenX] Generate output from base.xproto
"""
args = XOSProcessorArgs(files=[TEST_XPROTO], target=TEST_TARGET)
output = XOSProcessor.process(args)
self.assertEqual(output, TEST_EXPECTED_OUTPUT)
def test_singularize(self):
proto = """
message TestSingularize {
// The following field has an explicitly specified singular
required int many = 1 [singular = "one"];
// The following fields have automatically computed singulars
required int sheep = 2;
required int slices = 2;
required int networks = 2;
required int omf_friendlies = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages.0.fields -%}
{{ xproto_singularize(m) }},
{%- endfor %}
"""
)
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual(
"one,sheep,slice,network,omf_friendly", output.lstrip().rstrip().rstrip(",")
)
def test_django_with_attic(self):
"""
[XOS-GenX] Generate django output from test.xproto
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target="django.xtarget",
attic=TEST_ATTICS,
output=OUTPUT_DIR,
dest_extension="py",
write_to_file="model",
)
output = XOSProcessor.process(args)
# xosmodel has custom header attic
self.assertIn("from core.models.xosbase import *", output["XOSModel"])
self.assertIn("class XOSModel_decl(XOSBase):", output["XOSModel"])
# vrouter port use the default header
self.assertIn("from core.models.xosbase import *", output["VRouterPort"])
self.assertIn("class VRouterPort_decl(XOSBase):", output["VRouterPort"])
# verify files
xosmodel = OUTPUT_DIR + "/xosmodel.py"
self.assertTrue(os.path.isfile(xosmodel))
xmf = open(xosmodel).read()
self.assertIn("from core.models.xosbase import *", xmf)
self.assertIn("class XOSModel_decl(XOSBase):", xmf)
vrouterport = OUTPUT_DIR + "/vrouterport.py"
self.assertTrue(os.path.isfile(vrouterport))
vrpf = open(vrouterport).read()
self.assertIn("from core.models.xosbase import *", vrpf)
self.assertIn("class VRouterPort_decl(XOSBase):", vrpf)
def test_bin(self):
xproto = """
policy output < ctx.is_admin = True | obj.empty = True>
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals()) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
return i1
"""
obj = FakeObject()
obj.empty = True
ctx = FakeObject()
ctx.is_admin = True
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def _test_field_graph(self):
xproto = """
message VRouterDevice (PlCoreBase){
optional string name = 1 [help_text = "device friendly name", max_length = 20, null = True, db_index = False, blank = True, unique_with="openflow_id"];
required string openflow_id = 2 [help_text = "device identifier in ONOS", max_length = 20, null = False, db_index = False, blank = False, unique_with="name"];
required string config_key = 3 [default = "basic", max_length = 32, blank = False, help_text = "configuration key", null = False, db_index = False, unique_with="driver"];
required string driver = 4 [help_text = "driver type", max_length = 32, null = False, db_index = False, blank = False, unique_with="vrouter_service"];
required manytoone vrouter_service->VRouterService:devices = 5 [db_index = True, null = False, blank = False];
required string A = 6 [unique_with="B"];
required string B = 7 [unique_with="C"];
required string C = 8 [unique_with="A"];
required string D = 9;
required string E = 10 [unique_with="F,G"];
required string F = 11;
required string G = 12;
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
{{ xproto_field_graph_components(proto.messages.0.fields, proto.messages.0) }}
"""
)
args = XOSProcessorArgs(inputs=xproto, target=target)
output = XOSProcessor.process(args)
output = eval(output)
self.assertIn({"A", "B", "C"}, output)
self.assertIn({"openflow_id", "name"}, output)
self.assertIn({"config_key", "vrouter_service", "driver"}, output)
self.assertIn({"E", "F", "G"}, output)
union = reduce(lambda acc, x: acc | x, output)
self.assertNotIn("D", union)
def test_controller_network_policy(self):
xproto = """
policy test_policy <
ctx.user.is_admin
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Slice"
& Privilege.object_id = obj.owner.id)
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Site"
& Privilege.object_id = obj.owner.site.id
& Privilege.permission = "role:admin") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_enforcer(obj, ctx):
i2 = ctx.user.is_admin
i4 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Slice'), Q(object_id=obj.owner.id))[0]
i5 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Site'), Q(object_id=obj.owner.site.id), Q(permission='role:admin'))[0]
i3 = (i4 or i5)
i1 = (i2 or i3)
return i1
"""
# FIXME: Test this policy by executing it
self.assertTrue(policy_output_enforcer is not None)
def test_bin(self):
xproto = """
policy output < (ctx.is_admin = True | obj.empty = True) | False>
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output) # This loads the generated function, which should look like this:
"""
def policy_output_validator(obj, ctx):
i2 = (ctx.is_admin == True)
i3 = (obj.empty == True)
i1 = (i2 or i3)
if (not i1):
raise Exception('Necessary Failure')
"""
obj = FakeObject()
obj.empty = False
ctx = FakeObject()
ctx.is_admin = False
with self.assertRaises(Exception):
verdict = policy_output_validator(obj, ctx)
def test_filter_apps(self):
""" Should only get models whose apps are specified by include_apps """
args = XOSProcessorArgs(
files=[FILTERTEST_XPROTO], target=FILTERTEST_TARGET, include_apps=["core"]
)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model2,")
def test_base_class_fields(self):
target = XProtoTestHelpers.write_tmp_target(
"""
{% for m in proto.messages %}
{{ m.name }} {
{%- for b in m.bases %}
{%- if proto.message_table[b.fqn] -%}
{%- set model = proto.message_table[b.fqn] %}
{% for f in model.fields %}
{{ f.type }} {{ f.name }};
{% endfor %}
{%- endif -%}
{% endfor %}
}
{% endfor %}
"""
)
xproto = """
package xos.network;
message Port (PlCoreBase,ParameterMixin){
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
package xos.someotherpackage;
message Instance (xos.network.Port){
optional string instance_id = 1 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance id", null = True, db_index = False];
optional string instance_uuid = 2 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance uuid", null = True, db_index = False];
required string name = 3 [max_length = 200, content_type = "stripped", blank = False, help_text = "Instance name", null = False, db_index = False];
optional string instance_name = 4 [max_length = 200, content_type = "stripped", blank = True, help_text = "OpenStack generated name", null = True, db_index = False];
optional string ip = 5 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
required manytoone image->Image:instances = 6 [db_index = True, null = False, blank = False];
optional manytoone creator->User:instances = 7 [db_index = True, null = True, blank = True];
required manytoone slice->Slice:instances = 8 [db_index = True, null = False, blank = False];
required manytoone deployment->Deployment:instance_deployment = 9 [db_index = True, null = False, blank = False];
required manytoone node->Node:instances = 10 [db_index = True, null = False, blank = False];
required int32 numberCores = 11 [help_text = "Number of cores for instance", default = 0, null = False, db_index = False, blank = False];
required manytoone flavor->Flavor:instance = 12 [help_text = "Flavor of this instance", default = "get_default_flavor()", null = False, db_index = True, blank = False];
optional string userData = 13 [help_text = "user_data passed to instance during creation", null = True, db_index = False, blank = True];
required string isolation = 14 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
optional string volumes = 15 [help_text = "Comma-separated list of directories to expose to parent context", null = True, db_index = False, blank = True];
optional manytoone parent->Instance:instance = 16 [help_text = "Parent Instance for containers nested inside of VMs", null = True, db_index = True, blank = True];
required manytomany tags->Tag = 17 [db_index = False, null = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("xos_created", output)
def generate_swagger_docs(xproto):
# if not os.path.isfile(xproto):
# print "ERROR: Couldn't find xproto file for %s at: %s" % (service, xproto)
# return
print "Generating swagger docs for %s" % (xproto)
args = XOSProcessorArgs()
args.files = xproto
args.target = 'swagger.xtarget'
args.output = SWAGGER_DOCS_DIR
args.write_to_file = "single"
args.dest_file = "swagger.yaml"
args.quiet = False
try:
XOSProcessor.process(args)
except Exception:
print "ERROR: Couldn't generate swagger specs"
traceback.print_exc()
def test_filter_models(self):
""" Should only get models specified by include_models """
args = XOSProcessorArgs(
files=[FILTERTEST_XPROTO],
target=FILTERTEST_TARGET,
include_models=["Model1", "Model3"],
)
output = XOSProcessor.process(args)
self.assertEqual(output, "Model1,Model3,")
def test_xproto_model_to_oneof_key(self):
"""
[XOS-GenX] in some models we need to have a combine key on variable fields, for example, keys can be subscriber_service_id + oneof(provider_service_id, provider_network_id)
"""
xproto = """
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required string key_1 = 2 [ null = "False", blank="False", tosca_key_one_of = "key_2"];
required string key_2 = 3 [ null = "False", blank="False", tosca_key_one_of = "key_1"];
required string key_3 = 4 [ null = "False", blank="False", tosca_key_one_of = "key_4"];
required string key_4 = 5 [ null = "False", blank="False", tosca_key_one_of = "key_3"];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target_tosca_keys
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_1', 'key_2'], ['key_3', 'key_4']]", output)
xproto = """
option app_label = "test";
message Foo {
option tosca_key = "key1, oneof(key_2, key_3)";
required string name = 1 [ null = "False", blank="False"];
required manytoone key_1->Bar:key_1s = 2;
required manytoone key_2->Bar:key_2s = 3 [tosca_key_one_of = "key_1"];
required manytoone key_3->Bar:key_3s = 4 [tosca_key_one_of = "key_1"];
}
"""
args.inputs = xproto
output = XOSProcessor.process(args)
self.assertIn("['name', ['key_1_id', 'key_2_id', 'key_3_id']]", output)
def test_forall(self):
# This one we only parse
xproto = """
policy output < forall Credential: Credential.obj_id = obj_id >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(output, globals())
"""
def test_xproto_lib(self):
target = XProtoTestHelpers.write_tmp_target(
"""
{{ xproto_first_non_empty([None, None, None, None, None, None, "Eureka"]) }}
"""
)
args = XOSProcessorArgs()
args.inputs = ""
args.target = target
output = XOSProcessor.process(args)
self.assertIn("Eureka", output)
def test_proto_generator(self):
"""
[XOS-GenX] Generate DJANGO models, verify Fields and Foreign Keys
"""
args = XOSProcessorArgs(files=[VROUTER_XPROTO], target="django.xtarget")
output = XOSProcessor.process(args)
fields = [s for s in output.splitlines() if "Field(" in s]
self.assertEqual(len(fields), 2)
links = [s for s in output.splitlines() if "Key(" in s]
self.assertEqual(len(links), 2)
def test_write_multiple_files_from_xtarget(self):
"""
[XOS-GenX] read multiple models as input, print separate files based on +++
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target=SPLIT_TARGET,
output=OUTPUT_DIR,
write_to_file="target",
)
XOSProcessor.process(args)
generated_files = [f for f in os.listdir(OUTPUT_DIR) if not f.startswith(".")]
self.assertEqual(len(generated_files), 2)
xosmodel = open(os.path.join(OUTPUT_DIR, "xosmodel.txt"), "r").read()
vrouterport = open(os.path.join(OUTPUT_DIR, "vrouterport.txt"), "r").read()
self.assertIn("name: XOSModel", xosmodel)
self.assertIn("name: VRouterPort", vrouterport)
def test_call_policy(self):
xproto = \
"""
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = FakeArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output,globals()) # This loads the generated function, which should look like this:
"""
def sub_policy_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
def output_security_check(obj, ctx):
if obj.child:
i1 = sub_policy_security_check(obj.child, ctx)
else:
i1 = True
return i1
"""
obj = FakeArgs()
obj.child = FakeArgs()
obj.child.user = 1
ctx = FakeArgs()
ctx.user = 1
verdict = output_security_check(obj, ctx)
self.assertTrue(verdict)
def test_django_with_attic(self):
"""
[XOS-GenX] Generate django output from test.xproto
"""
args = XOSProcessorArgs(
files=[TEST_XPROTO, VROUTER_XPROTO],
target="django.xtarget",
attic=TEST_ATTICS,
output=OUTPUT_DIR,
dest_extension="py",
write_to_file="model",
)
output = XOSProcessor.process(args)
# xosmodel has custom header attic
self.assertIn("from core.models.xosbase import *", output["XOSModel"])
self.assertIn("class XOSModel_decl(XOSBase):", output["XOSModel"])
# vrouter port use the default header
self.assertIn("from core.models.xosbase import *",
output["VRouterPort"])
self.assertIn("class VRouterPort_decl(XOSBase):",
output["VRouterPort"])
# verify files
xosmodel = OUTPUT_DIR + "/xosmodel.py"
self.assertTrue(os.path.isfile(xosmodel))
xmf = open(xosmodel).read()
self.assertIn("from core.models.xosbase import *", xmf)
self.assertIn("class XOSModel_decl(XOSBase):", xmf)
vrouterport = OUTPUT_DIR + "/vrouterport.py"
self.assertTrue(os.path.isfile(vrouterport))
vrpf = open(vrouterport).read()
self.assertIn("from core.models.xosbase import *", vrpf)
self.assertIn("class VRouterPort_decl(XOSBase):", vrpf)
def test_field_numbers(self):
args = XOSProcessorArgs(files=[REVERSEFIELDTEST_XPROTO],
target=FIELDTEST_TARGET)
output = XOSProcessor.process(args)
def _assert_field(modelname, fieldname, id):
self.assertIn("%s,%s,%s" % (modelname, fieldname, id), output)
# rel_int1s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1as_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1a", "rel_int1s_ids", 1001)
_assert_field("Leaf1a", "rel_leaf1as_ids", 1101)
# rel_int2s_ids is the reverse link from RelatedToIntermediate1. It gets the related id with no offset, so it
# will be assigned 1001. rel_leaf1bs_ids inherits from Intermediate1, so its reverse links will all be offset
# by 100
_assert_field("Leaf1b", "rel_int1s_ids", 1001)
_assert_field("Leaf1b", "rel_leaf1bs_ids", 1101)
# There are no reverse numbers specified for Intermediate2 or Leaf2, so xproto will fall back to automatic
# numbering starting at 1900.
_assert_field("Leaf2", "rel_int2s_ids", 1900)
_assert_field("Leaf2", "rel_leaf2s_ids", 1901)
def test_controller_network_policy(self):
xproto = """
policy test_policy <
ctx.user.is_admin
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Slice"
& Privilege.object_id = obj.owner.id)
| (exists Privilege:
Privilege.accessor_id = ctx.user.id
& Privilege.accessor_type = "User"
& Privilege.object_type = "Site"
& Privilege.object_id = obj.owner.site.id
& Privilege.permission = "role:admin") >
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def policy_output_enforcer(obj, ctx):
i2 = ctx.user.is_admin
i4 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Slice'), Q(object_id=obj.owner.id))[0]
i5 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(accessor_type='User'), Q(object_type='Site'), Q(object_id=obj.owner.site.id), Q(permission='role:admin'))[0]
i3 = (i4 or i5)
i1 = (i2 or i3)
return i1
"""
# FIXME: Test this policy by executing it
self.assertTrue(policy_output_enforcer is not None)
def test_equal(self):
xproto = """
policy output < ctx.user = obj.user >
"""
args = XOSProcessorArgs(inputs=xproto, target=self.target)
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def output_security_check(obj, ctx):
i1 = (ctx.user == obj.user)
return i1
"""
obj = FakeObject()
obj.user = 1
ctx = FakeObject()
ctx.user = 1
verdict = output_security_check(obj, ctx)
def test_exists(self):
xproto = \
"""
policy privilege < exists Privilege: Privilege.object_id = obj.id >
"""
target = XProtoTestHelpers.write_tmp_target(
"{{ proto.policies.privilege }} ")
args = FakeArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
Privilege = FakeArgs()
Privilege.object_id = 1
obj = FakeArgs()
obj.id = 1
(op, operands), = eval(output).items()
(op2, operands2), = operands[1].items()
expr = op2.join(operands2).replace('=', '==')
self.assertTrue(eval(expr))
def test_singularize(self):
proto = """
message TestSingularize {
// The following field has an explicitly specified singular
required int many = 1 [singular = "one"];
// The following fields have automatically computed singulars
required int sheep = 2;
required int slices = 2;
required int networks = 2;
required int omf_friendlies = 2;
}
"""
target = XProtoTestHelpers.write_tmp_target("""
{% for m in proto.messages.0.fields -%}
{{ xproto_singularize(m) }},
{%- endfor %}
""")
args = XOSProcessorArgs()
args.inputs = proto
args.target = target
output = XOSProcessor.process(args)
self.assertEqual("one,sheep,slice,network,omf_friendly",
output.lstrip().rstrip().rstrip(","))
def test_optional_relations(self):
"""
[XOS-GenX] Generate DJANGO models, verify relations
"""
xproto = \
"""
option app_label = "test";
message ENodeB {
}
message Handover {
}
message Foo {
optional manytoone enodeb->ENodeB:profiles = 1 [null = True, blank = True];
required manytoone handover->Handover:profiles = 2 [null = False, blank = False];
}
"""
args = FakeArgs()
args.inputs = xproto
args.target = 'django.xtarget'
output = XOSProcessor.process(args)
null_true = filter(lambda s: 'null = True' in s, output.splitlines())
null_false = filter(lambda s: 'null = False' in s, output.splitlines())
blank_true = filter(lambda s: 'blank = True' in s, output.splitlines())
blank_false = filter(lambda s: 'blank = False' in s,
output.splitlines())
self.assertEqual(len(null_true), 1)
self.assertEqual(len(null_false), 1)
self.assertEqual(len(blank_true), 1)
self.assertEqual(len(blank_false), 1)
def test_controller_policy(self):
xproto = \
"""
policy test_policy < ctx.user.is_admin | exists Privilege: Privilege.accessor_id = ctx.user.id & Privilege.object_type = "Deployment" & Privilege.permission = "role:admin" & Privilege.object_id = obj.id >
"""
args = FakeArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(
output
) # This loads the generated function, which should look like this:
"""
def policy_output_enforcer(obj, ctx):
i2 = ctx.user.is_admin
i3 = Privilege.objects.filter(Q(accessor_id=ctx.user.id), Q(object_type='Deployment'), Q(permission='role:admin'), Q(object_id=obj.id))[0]
i1 = (i2 or i3)
return i1
"""
# FIXME: Test this policy by executing it
self.assertTrue(policy_output_enforcer is not None)
def test_call_policy(self):
xproto = \
"""
policy sub_policy < ctx.user = obj.user >
policy output < *sub_policy(child) >
"""
args = FakeArgs()
args.inputs = xproto
args.target = self.target
output = XOSProcessor.process(args)
exec(output, globals(
)) # This loads the generated function, which should look like this:
"""
def policy_sub_policy_validator(obj, ctx):
i1 = (ctx.user == obj.user)
if (not i1):
raise ValidationError('Necessary Failure')
def policy_output_validator(obj, ctx):
i1 = policy_sub_policy_validator(obj.child, ctx)
if (not i1):
raise ValidationError('Necessary Failure')
"""
obj = FakeArgs()
obj.child = FakeArgs()
obj.child.user = 1
ctx = FakeArgs()
ctx.user = 1
with self.assertRaises(Exception):
verdict = policy_output_enforcer(obj, ctx)
def test_pure_proto(self):
xproto = """
message VRouterPort (XOSBase){
optional string name = 1 [help_text = "port friendly name", max_length = 20, null = True, db_index = False, blank = True];
required string openflow_id = 2 [help_text = "port identifier in ONOS", max_length = 21, null = False, db_index = False, blank = False];
required manytoone vrouter_device->VRouterDevice:ports = 3 [db_index = True, null = False, blank = False];
required manytoone vrouter_service->VRouterService:device_ports = 4 [db_index = True, null = False, blank = False];
}
"""
proto = """
message VRouterPort {
option bases = "XOSBase";
optional string name = 1 [ null = "True", max_length = "20", blank = "True", help_text = "port friendly name", modifier = "optional", db_index = "False" ];
required string openflow_id = 2 [ null = "False", max_length = "21", blank = "False", help_text = "port identifier in ONOS", modifier = "required", db_index = "False" ];
required int32 vrouter_device = 3 [ null = "False", blank = "False", model = "VRouterDevice", modifier = "required", type = "link", port = "ports", db_index = "True", link = "manytoone"];
required int32 vrouter_service = 4 [ null = "False", blank = "False", model = "VRouterService", modifier = "required", type = "link", port = "device_ports", db_index = "True", link = "manytoone"];
}
"""
target = XProtoTestHelpers.write_tmp_target(
"""
from header import *
{% for m in proto.messages %}
{% if file_exists(xproto_base_name(m.name)|lower+'_header.py') -%}from {{xproto_base_name(m.name)|lower }}_header import *{% endif %}
{% if file_exists(xproto_base_name(m.name)|lower+'_top.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_top.py') }} {% endif %}
{%- for l in m.links %}
{% if l.peer.name != m.name %}
from core.models.{{ l.peer.name | lower }} import {{ l.peer.name }}
{% endif %}
{%- endfor %}
{% for b in m.bases %}
{% if b!='XOSBase' and 'Mixin' not in b%}
from core.models.{{b.name | lower}} import {{ b.name }}
{% endif %}
{% endfor %}
class {{ m.name }}{{ xproto_base_def(m, m.bases) }}:
# Primitive Fields (Not Relations)
{% for f in m.fields %}
{%- if not f.link -%}
{{ f.name }} = {{ xproto_django_type(f.type, f.options) }}( {{ xproto_django_options_str(f) }} )
{% endif %}
{%- endfor %}
# Relations
{% for l in m.links %}
{{ l.src_port }} = {{ xproto_django_link_type(l) }}( {%- if l.peer.name==m.name -%}'self'{%- else -%}{{ l.peer.name }} {%- endif -%}, {{ xproto_django_link_options_str(l, l.dst_port ) }} )
{%- endfor %}
{% if file_exists(m.name|lower + '_model.py') -%}{{ include_file(m.name|lower + '_model.py') | indent(width=2)}}{%- endif %}
pass
{% if file_exists(xproto_base_name(m.name)|lower+'_bottom.py') -%}{{ include_file(xproto_base_name(m.name)|lower+'_bottom.py') }}{% endif %}
{% endfor %}
"""
)
args_xproto = XOSProcessorArgs()
args_xproto.inputs = xproto
args_xproto.target = target
xproto_gen = XOSProcessor.process(args_xproto)
count1 = len(xproto_gen.split("\n"))
args_proto = XOSProcessorArgs()
args_proto.inputs = proto
args_proto.target = target
args_proto.rev = True
proto_gen = XOSProcessor.process(args_proto)
count2 = len(proto_gen.split("\n"))
self.assertEqual(count1, count2)
def run_xosgenx_service(self, manifest):
if not os.path.exists(manifest["dest_dir"]):
os.makedirs(manifest["dest_dir"])
xproto_filenames = [
os.path.join(manifest["dir"], x["filename"])
for x in manifest["xprotos"]
]
class Args:
pass
# Generate models
is_service = manifest["name"] != 'core'
args = Args()
args.output = manifest["dest_dir"]
args.attic = os.path.join(manifest["dir"], 'attic')
args.files = xproto_filenames
if is_service:
args.target = 'service.xtarget'
args.write_to_file = 'target'
else:
args.target = 'django.xtarget'
args.dest_extension = 'py'
args.write_to_file = 'model'
XOSProcessor.process(args)
# Generate security checks
class SecurityArgs:
output = manifest["dest_dir"]
target = 'django-security.xtarget'
dest_file = 'security.py'
write_to_file = 'single'
files = xproto_filenames
XOSProcessor.process(SecurityArgs())
# Generate __init__.py
if manifest["name"] == "core":
class InitArgs:
output = manifest["dest_dir"]
target = 'init.xtarget'
dest_file = '__init__.py'
write_to_file = 'single'
files = xproto_filenames
XOSProcessor.process(InitArgs())
else:
init_py_filename = os.path.join(manifest["dest_dir"],
"__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# the xosgenx templates don't handle copying the models.py file for us, so do it here.
for item in manifest["decls"]:
src_fn = os.path.join(manifest["dir"], item["filename"])
dest_fn = os.path.join(manifest["dest_dir"], item["filename"])
shutil.copyfile(src_fn, dest_fn)
# If the attic has a header.py, make sure it is copied to the right place
attic_header_py_src = os.path.join(manifest["dir"], "attic",
"header.py")
service_header_py_dest = os.path.join(manifest["dest_dir"],
"header.py")
if os.path.exists(attic_header_py_src):
shutil.copyfile(attic_header_py_src, service_header_py_dest)
elif os.path.exists(service_header_py_dest):
os.remove(service_header_py_dest)
def test_yaml_generator(self):
xproto = \
"""
option app_label = "test";
message Port (PlCoreBase,ParameterMixin){
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
message Instance (PlCoreBase){
optional string instance_id = 1 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance id", null = True, db_index = False];
optional string instance_uuid = 2 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance uuid", null = True, db_index = False];
required string name = 3 [max_length = 200, content_type = "stripped", blank = False, help_text = "Instance name", null = False, db_index = False];
optional string instance_name = 4 [max_length = 200, content_type = "stripped", blank = True, help_text = "OpenStack generated name", null = True, db_index = False];
optional string ip = 5 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
required manytoone image->Image:instances = 6 [db_index = True, null = False, blank = False];
optional manytoone creator->User:instances = 7 [db_index = True, null = True, blank = True];
required manytoone slice->Slice:instances = 8 [db_index = True, null = False, blank = False];
required manytoone deployment->Deployment:instance_deployment = 9 [db_index = True, null = False, blank = False];
required manytoone node->Node:instances = 10 [db_index = True, null = False, blank = False];
required int32 numberCores = 11 [help_text = "Number of cores for instance", default = 0, null = False, db_index = False, blank = False];
required manytoone flavor->Flavor:instance = 12 [help_text = "Flavor of this instance", default = "get_default_flavor()", null = False, db_index = True, blank = False];
optional string userData = 13 [help_text = "user_data passed to instance during creation", null = True, db_index = False, blank = True];
required string isolation = 14 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
optional string volumes = 15 [help_text = "Comma-separated list of directories to expose to parent context", null = True, db_index = False, blank = True];
optional manytoone parent->Instance:instance = 16 [help_text = "Parent Instance for containers nested inside of VMs", null = True, db_index = True, blank = True];
required manytomany tags->Tag = 17 [db_index = False, null = False, blank = True];
}
message Network (PlCoreBase,ParameterMixin) {
required string name = 1 [db_index = False, max_length = 32, null = False, blank = False];
required manytoone template->NetworkTemplate:network = 2 [db_index = True, null = False, blank = False];
required string subnet = 3 [db_index = False, max_length = 32, null = False, blank = True];
required string start_ip = 4 [db_index = False, max_length = 32, null = False, blank = True];
required string end_ip = 5 [db_index = False, max_length = 32, null = False, blank = True];
optional string ports = 6 [db_index = False, max_length = 1024, null = True, blank = True];
optional string labels = 7 [db_index = False, max_length = 1024, null = True, blank = True];
required manytoone owner->Slice:ownedNetworks = 8 [help_text = "Slice that owns control of this Network", null = False, db_index = True, blank = False];
required int32 guaranteed_bandwidth = 9 [default = 0, null = False, db_index = False, blank = False];
required bool permit_all_slices = 10 [default = False, null = False, db_index = False, blank = True];
optional string topology_parameters = 11 [db_index = False, null = True, blank = True];
optional string controller_url = 12 [db_index = False, max_length = 1024, null = True, blank = True];
optional string controller_parameters = 13 [db_index = False, null = True, blank = True];
optional string network_id = 14 [help_text = "Quantum network", max_length = 256, null = True, db_index = False, blank = True];
optional string router_id = 15 [help_text = "Quantum router id", max_length = 256, null = True, db_index = False, blank = True];
optional string subnet_id = 16 [help_text = "Quantum subnet id", max_length = 256, null = True, db_index = False, blank = True];
required bool autoconnect = 17 [help_text = "This network can be autoconnected to the slice that owns it", default = True, null = False, db_index = False, blank = True];
required manytomany permitted_slices->Slice/Network_permitted_slices:availableNetworks = 18 [db_index = False, null = False, blank = True];
required manytomany slices->Slice/NetworkSlice:networks = 19 [db_index = False, null = False, blank = True];
required manytomany instances->Instance/Port:networks = 20 [db_index = False, null = False, blank = True];
}
message Slice (PlCoreBase){
required string name = 1 [max_length = 80, content_type = "stripped", blank = False, help_text = "The Name of the Slice", null = False, db_index = False];
required bool enabled = 2 [help_text = "Status for this Slice", default = True, null = False, db_index = False, blank = True];
required bool omf_friendly = 3 [default = False, null = False, db_index = False, blank = True];
required string description = 4 [help_text = "High level description of the slice and expected activities", max_length = 1024, null = False, db_index = False, blank = True];
required string slice_url = 5 [db_index = False, max_length = 512, null = False, content_type = "url", blank = True];
required manytoone site->Site:slices = 6 [help_text = "The Site this Slice belongs to", null = False, db_index = True, blank = False];
required int32 max_instances = 7 [default = 10, null = False, db_index = False, blank = False];
optional manytoone service->Service:slices = 8 [db_index = True, null = True, blank = True];
optional string network = 9 [blank = True, max_length = 256, null = True, db_index = False, choices = "((None, 'Default'), ('host', 'Host'), ('bridged', 'Bridged'), ('noauto', 'No Automatic Networks'))"];
optional string exposed_ports = 10 [db_index = False, max_length = 256, null = True, blank = True];
optional manytoone serviceClass->ServiceClass:slices = 11 [db_index = True, null = True, blank = True];
optional manytoone creator->User:slices = 12 [db_index = True, null = True, blank = True];
optional manytoone default_flavor->Flavor:slices = 13 [db_index = True, null = True, blank = True];
optional manytoone default_image->Image:slices = 14 [db_index = True, null = True, blank = True];
optional manytoone default_node->Node:slices = 15 [db_index = True, null = True, blank = True];
optional string mount_data_sets = 16 [default = "GenBank", max_length = 256, content_type = "stripped", blank = True, null = True, db_index = False];
required string default_isolation = 17 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
required manytomany tags->Tag = 18 [db_index = False, null = False, blank = True];
}
"""
args = FakeArgs()
args.inputs = xproto
args.target = 'modeldefs.xtarget'
output = XOSProcessor.process(args)
yaml_ir = yaml.load(output)
self.assertEqual(len(yaml_ir['items']), 4)
def _test_proto_generator(self):
args = FakeArgs()
args.files = [VROUTER_XPROTO]
args.target = 'proto.xtarget'
output = XOSProcessor.process(args)
self.assertEqual(output, PROTO_EXPECTED_OUTPUT)
def run_xosgenx_service(self, manifest):
if not os.path.exists(manifest["dest_dir"]):
os.makedirs(manifest["dest_dir"])
xproto_filenames = [
os.path.join(manifest["dir"], x["filename"])
for x in manifest["xprotos"]
]
# Generate models
is_service = manifest["name"] != "core"
args = XOSProcessorArgs(
output=manifest["dest_dir"],
attic=os.path.join(manifest["dir"], "attic"),
files=xproto_filenames,
)
if is_service:
args.target = "service.xtarget"
args.write_to_file = "target"
else:
args.target = "django.xtarget"
args.dest_extension = "py"
args.write_to_file = "model"
XOSProcessor.process(args)
# Generate security checks
security_args = XOSProcessorArgs(
output=manifest["dest_dir"],
target="django-security.xtarget",
dest_file="security.py",
write_to_file="single",
files=xproto_filenames,
)
XOSProcessor.process(security_args)
# Generate __init__.py
if manifest["name"] == "core":
class InitArgs:
output = manifest["dest_dir"]
target = "init.xtarget"
dest_file = "__init__.py"
write_to_file = "single"
files = xproto_filenames
XOSProcessor.process(InitArgs())
else:
init_py_filename = os.path.join(manifest["dest_dir"],
"__init__.py")
if not os.path.exists(init_py_filename):
open(init_py_filename, "w").write("# created by dynamicbuild")
# the xosgenx templates don't handle copying the models.py file for us, so do it here.
for item in manifest["decls"]:
src_fn = os.path.join(manifest["dir"], item["filename"])
dest_fn = os.path.join(manifest["dest_dir"], item["filename"])
shutil.copyfile(src_fn, dest_fn)
# If the attic has a header.py, make sure it is copied to the right place
attic_header_py_src = os.path.join(manifest["dir"], "attic",
"header.py")
service_header_py_dest = os.path.join(manifest["dest_dir"],
"header.py")
if os.path.exists(attic_header_py_src):
shutil.copyfile(attic_header_py_src, service_header_py_dest)
elif os.path.exists(service_header_py_dest):
os.remove(service_header_py_dest)
def build(self):
# Destroy anything in the old build directory
if os.path.exists(BUILD_DIR):
for dir in os.listdir(BUILD_DIR):
shutil.rmtree(os.path.join(BUILD_DIR, dir))
# Copy all of the resources into the build directory
for (kind, src_fn, dest_fn, service_name) in self.resources:
build_dest_fn = os.path.join(BUILD_DIR, dest_fn)
makedirs_if_noexist(os.path.dirname(build_dest_fn))
if (os.path.isdir(src_fn)):
if (not os.path.isdir(build_dest_fn)):
shutil.copytree(src_fn, build_dest_fn, symlinks=True)
else:
os.system(
'cp -R %(src_fn)s/*.xproto %(src_fn)s/attic %(src_fn)s/models.py %(src_fn)s/*header.py %(build_dst_fn)s 2> /dev/null || :'
% {
'src_fn': src_fn,
'build_dst_fn': build_dest_fn
})
else:
shutil.copyfile(src_fn, build_dest_fn)
if (kind == 'xproto'):
xprotos = [
f for f in os.listdir(src_fn) if f.endswith('xproto')
]
file_list = []
for x in xprotos:
file_list.append(os.path.join(src_fn, x))
try:
class Args:
pass
# Generate models
is_service = service_name != 'core'
args = Args()
args.output = build_dest_fn
args.attic = src_fn + '/attic'
args.files = file_list
if is_service:
args.target = 'service.xtarget'
args.write_to_file = 'target'
else:
args.target = 'django.xtarget'
args.dest_extension = 'py'
args.write_to_file = 'model'
XOSProcessor.process(args)
# Generate security checks
class SecurityArgs:
output = build_dest_fn
target = 'django-security.xtarget'
dest_file = 'security.py'
write_to_file = 'single'
files = file_list
XOSProcessor.process(SecurityArgs())
# Generate __init__.py
if service_name == "core":
class InitArgs:
output = build_dest_fn
target = 'init.xtarget'
dest_file = '__init__.py'
write_to_file = 'single'
files = file_list
XOSProcessor.process(InitArgs())
except Exception, e:
print 'xproto build failed.'
raise e
def generate():
args = FakeArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
def test_cross_model(self):
target = XProtoTestHelpers.write_tmp_target("""
{% for m in proto.messages %}
{{ m.fqn }} {
{%- for l in m.links %}
{%- if proto.message_table[l.peer.fqn] %}
{{ l.peer.name }} {
{%- set model = proto.message_table[l.peer.fqn] %}
{% for f in model.fields %}
{{ f.type }} {{ f.name }};
{% endfor %}
}
{%- endif -%}
{%- if proto.message_table[m.package + '.' + l.peer.name] %}
{{ l.peer.name }} {
{%- set model = proto.message_table[m.package + '.' + l.peer.name] %}
{% for f in model.fields %}
{{ f.type }} {{ f.name }};
{% endfor %}
}
{%- endif -%}
{% endfor %}
}
{% endfor %}
""")
xproto = """
package xos.network;
message Port (PlCoreBase,ParameterMixin){
required manytoone network->Network:links = 1 [db_index = True, null = False, blank = False];
optional manytoone instance->xos.core.Instance:ports = 2 [db_index = True, null = True, blank = True];
optional string ip = 3 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
optional string port_id = 4 [help_text = "Neutron port id", max_length = 256, null = True, db_index = False, blank = True];
optional string mac = 5 [help_text = "MAC address associated with this port", max_length = 256, null = True, db_index = False, blank = True];
required bool xos_created = 6 [default = False, null = False, db_index = False, blank = True];
}
package xos.core;
message Instance (PlCoreBase){
optional string instance_id = 1 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance id", null = True, db_index = False];
optional string instance_uuid = 2 [max_length = 200, content_type = "stripped", blank = True, help_text = "Nova instance uuid", null = True, db_index = False];
required string name = 3 [max_length = 200, content_type = "stripped", blank = False, help_text = "Instance name", null = False, db_index = False];
optional string instance_name = 4 [max_length = 200, content_type = "stripped", blank = True, help_text = "OpenStack generated name", null = True, db_index = False];
optional string ip = 5 [max_length = 39, content_type = "ip", blank = True, help_text = "Instance ip address", null = True, db_index = False];
required manytoone image->Image:instances = 6 [db_index = True, null = False, blank = False];
optional manytoone creator->User:instances = 7 [db_index = True, null = True, blank = True];
required manytoone slice->Slice:instances = 8 [db_index = True, null = False, blank = False];
required manytoone deployment->Deployment:instance_deployment = 9 [db_index = True, null = False, blank = False];
required manytoone node->Node:instances = 10 [db_index = True, null = False, blank = False];
required int32 numberCores = 11 [help_text = "Number of cores for instance", default = 0, null = False, db_index = False, blank = False];
required manytoone flavor->Flavor:instance = 12 [help_text = "Flavor of this instance", default = "get_default_flavor()", null = False, db_index = True, blank = False];
optional string userData = 13 [help_text = "user_data passed to instance during creation", null = True, db_index = False, blank = True];
required string isolation = 14 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
optional string volumes = 15 [help_text = "Comma-separated list of directories to expose to parent context", null = True, db_index = False, blank = True];
optional manytoone parent->Instance:instance = 16 [help_text = "Parent Instance for containers nested inside of VMs", null = True, db_index = True, blank = True];
required manytomany tags->Tag = 17 [db_index = False, null = False, blank = True];
}
package xos.network;
message Network (PlCoreBase,ParameterMixin) {
required string name = 1 [db_index = False, max_length = 32, null = False, blank = False];
required manytoone template->NetworkTemplate:network = 2 [db_index = True, null = False, blank = False];
required string subnet = 3 [db_index = False, max_length = 32, null = False, blank = True];
required string start_ip = 4 [db_index = False, max_length = 32, null = False, blank = True];
required string end_ip = 5 [db_index = False, max_length = 32, null = False, blank = True];
optional string ports = 6 [db_index = False, max_length = 1024, null = True, blank = True];
optional string labels = 7 [db_index = False, max_length = 1024, null = True, blank = True];
required manytoone owner->Slice:ownedNetworks = 8 [help_text = "Slice that owns control of this Network", null = False, db_index = True, blank = False];
required int32 guaranteed_bandwidth = 9 [default = 0, null = False, db_index = False, blank = False];
required bool permit_all_slices = 10 [default = False, null = False, db_index = False, blank = True];
optional string topology_parameters = 11 [db_index = False, null = True, blank = True];
optional string controller_url = 12 [db_index = False, max_length = 1024, null = True, blank = True];
optional string controller_parameters = 13 [db_index = False, null = True, blank = True];
optional string network_id = 14 [help_text = "Quantum network", max_length = 256, null = True, db_index = False, blank = True];
optional string router_id = 15 [help_text = "Quantum router id", max_length = 256, null = True, db_index = False, blank = True];
optional string subnet_id = 16 [help_text = "Quantum subnet id", max_length = 256, null = True, db_index = False, blank = True];
required bool autoconnect = 17 [help_text = "This network can be autoconnected to the slice that owns it", default = True, null = False, db_index = False, blank = True];
required manytomany permitted_slices->Slice/Network_permitted_slices:availableNetworks = 18 [db_index = False, null = False, blank = True];
required manytomany slices->Slice/NetworkSlice:networks = 19 [db_index = False, null = False, blank = True];
required manytomany instances->xos.core.Instance/xos.network.Port:networks = 20 [db_index = False, null = False, blank = True];
}
message Slice (PlCoreBase){
required string name = 1 [max_length = 80, content_type = "stripped", blank = False, help_text = "The Name of the Slice", null = False, db_index = False];
required bool enabled = 2 [help_text = "Status for this Slice", default = True, null = False, db_index = False, blank = True];
required bool omf_friendly = 3 [default = False, null = False, db_index = False, blank = True];
required string description = 4 [help_text = "High level description of the slice and expected activities", max_length = 1024, null = False, db_index = False, blank = True];
required string slice_url = 5 [db_index = False, max_length = 512, null = False, content_type = "url", blank = True];
required manytoone site->Site:slices = 6 [help_text = "The Site this Slice belongs to", null = False, db_index = True, blank = False];
required int32 max_instances = 7 [default = 10, null = False, db_index = False, blank = False];
optional manytoone service->Service:slices = 8 [db_index = True, null = True, blank = True];
optional string network = 9 [blank = True, max_length = 256, null = True, db_index = False, choices = "((None, 'Default'), ('host', 'Host'), ('bridged', 'Bridged'), ('noauto', 'No Automatic Networks'))"];
optional string exposed_ports = 10 [db_index = False, max_length = 256, null = True, blank = True];
optional manytoone serviceClass->ServiceClass:slices = 11 [db_index = True, null = True, blank = True];
optional manytoone creator->User:slices = 12 [db_index = True, null = True, blank = True];
optional manytoone default_flavor->Flavor:slices = 13 [db_index = True, null = True, blank = True];
optional manytoone default_image->Image:slices = 14 [db_index = True, null = True, blank = True];
optional manytoone default_node->Node:slices = 15 [db_index = True, null = True, blank = True];
optional string mount_data_sets = 16 [default = "GenBank", max_length = 256, content_type = "stripped", blank = True, null = True, db_index = False];
required string default_isolation = 17 [default = "vm", choices = "(('vm', 'Virtual Machine'), ('container', 'Container'), ('container_vm', 'Container In VM'))", max_length = 30, blank = False, null = False, db_index = False];
required manytomany tags->Tag = 18 [db_index = False, null = False, blank = True];
}
"""
args = XOSProcessorArgs()
args.inputs = xproto
args.target = target
output = XOSProcessor.process(args)
self.assertIn("numberCores",
output) # Instance showed up via cross-package call
self.assertIn("ip;",
output) # Network showed up via cross-package call
self.assertIn("max_instances",
output) # Slice showed up via implicit in-package call
def generate():
args = XOSProcessorArgs(inputs=xproto, target=target)
output = XOSProcessor.process(args)
"models", "core.xproto")
TARGET = os.path.join(BASE_DIR, "orchestration", "xos", "lib", "xos-genx",
"xosgenx", "targets", "fieldlist.xtarget")
def get_all_xproto():
xprotos = []
for service_name in os.listdir(SERVICES_DIR):
if service_name.startswith("."):
continue
service_path = os.path.join(SERVICES_DIR, service_name)
if not os.path.isdir(service_path):
continue
models_dir = os.path.join(service_path, "xos", "synchronizer",
"models")
if not os.path.isdir(models_dir):
continue
for xproto_name in os.listdir(models_dir):
if xproto_name.startswith("."):
continue
if not xproto_name.endswith(".xproto"):
continue
xproto_pathname = os.path.join(models_dir, xproto_name)
xprotos.append(xproto_pathname)
return xprotos
xprotos = get_all_xproto() + [CORE_XPROTO]
args = XOSProcessorArgs(files=xprotos, target=TARGET, verbosity=1)
output = XOSProcessor.process(args)