def handle_properties(self):
tosca_props = self.get_tosca_props()
self.log.debug(_("Port {0} with tosca properties: {1}").
format(self.name, tosca_props))
port_props = {}
for key, value in tosca_props.items():
port_props[key] = value
if 'cp_type' not in port_props:
port_props['cp_type'] = 'VPORT'
else:
if not port_props['cp_type'] in ToscaNetworkPort.VALID_TYPES:
err_msg = _("Invalid port type, {0}, specified for {1}"). \
format(port_props['cp_type'], self.name)
self.log.warn(err_msg)
raise ValidationError(message=err_msg)
if 'vdu_intf_type' not in port_props:
port_props['vdu_intf_type'] = 'VIRTIO'
else:
if not port_props['vdu_intf_type'] in ToscaNetworkPort.VALID_TYPES:
err_msg = _("Invalid port type, {0}, specified for {1}"). \
format(port_props['vdu_intf_type'], self.name)
self.log.warn(err_msg)
raise ValidationError(message=err_msg)
self.properties = port_props
def handle_capabilities(self):
def get_vm_flavor(specs):
vm_flavor = {}
if 'num_cpus' in specs:
vm_flavor['vcpu-count'] = specs['num_cpus']
else:
vm_flavor['vcpu-count'] = 1
if 'mem_size' in specs:
vm_flavor['memory-mb'] = (ScalarUnit_Size(specs['mem_size']).
get_num_from_scalar_unit('MB'))
else:
vm_flavor['memory-mb'] = 512
if 'disk_size' in specs:
vm_flavor['storage-gb'] = (ScalarUnit_Size(specs['disk_size']).
get_num_from_scalar_unit('GB'))
else:
vm_flavor['storage-gb'] = 4
return vm_flavor
tosca_caps = self.get_tosca_caps()
self.log.debug(_("VDU {0} tosca capabilites: {1}").
format(self.name, tosca_caps))
if 'host' in tosca_caps:
self.properties['vm-flavor'] = get_vm_flavor(tosca_caps['host'])
self.log.debug(_("VDU {0} properties: {1}").
format(self.name, self.properties))
def _create_archive(self, desc_id, output_dir):
"""Create a tar.gz archive for the descriptor"""
aname = desc_id + '.tar.gz'
apath = os.path.join(output_dir, aname)
self.log.debug(_("Generating archive: {}").format(apath))
prevdir = os.getcwd()
os.chdir(output_dir)
# Generate the archive
tar_cmd = "tar zcvf {} {}".format(apath, desc_id)
self.log.debug(_("Generate archive: {}").format(tar_cmd))
try:
subprocess.check_call(tar_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
return apath
except subprocess.CalledProcessError as e:
msg = _("Error creating archive with {}: {}"). \
format(tar_cmd, e)
self.log.error(msg)
raise ToscaCreateArchiveError(msg)
finally:
os.chdir(prevdir)
def _translate(self, sourcetype, parsed_params):
output = None
# Check the input file
path = os.path.abspath(self.template_file)
self.in_file = path
a_file = os.path.isfile(path)
if not a_file:
msg = _("The path {} is not a valid file."). \
format(self.template_file)
self.log.error(msg)
raise ValueError(msg)
# Get the file type
self.ftype = self._get_file_type()
self.log.debug(
_("Input file {0} is of type {1}").format(path, self.ftype))
if sourcetype == "tosca":
entry_file = self.get_entry_file()
if entry_file:
self.log.debug(_('Loading the tosca template.'))
tosca = ToscaTemplate(entry_file, parsed_params, True)
self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__))
translator = TOSCATranslator(self.log,
tosca,
parsed_params,
use_gi=self.use_gi)
self.log.debug(_('Translating the tosca template.'))
output = translator.translate()
return output
def handle_properties(self):
tosca_props = self.get_tosca_props()
self.log.debug(
_("Port {0} with tosca properties: {1}").format(
self.name, tosca_props))
port_props = {}
for key, value in tosca_props.items():
port_props[key] = value
if 'cp_type' not in port_props:
port_props['cp_type'] = 'VPORT'
else:
if not port_props['cp_type'] in ToscaNetworkPort.VALID_TYPES:
err_msg = _("Invalid port type, {0}, specified for {1}"). \
format(port_props['cp_type'], self.name)
self.log.warn(err_msg)
raise ValidationError(message=err_msg)
if 'vdu_intf_type' not in port_props:
port_props['vdu_intf_type'] = 'VIRTIO'
else:
if not port_props['vdu_intf_type'] in ToscaNetworkPort.VALID_TYPES:
err_msg = _("Invalid port type, {0}, specified for {1}"). \
format(port_props['vdu_intf_type'], self.name)
self.log.warn(err_msg)
raise ValidationError(message=err_msg)
self.cp_name = port_props['name']
self.properties = port_props
def handle_artifacts(self):
if self.artifacts is None:
return
self.log.debug(_("VDU {0} tosca artifacts: {1}").
format(self.name, self.artifacts))
arts = {}
for key in self.artifacts:
props = self.artifacts[key]
if isinstance(props, dict):
details = {}
for name, value in props.items():
if name == 'type':
prefix, type_ = value.rsplit('.', 1)
if type_ == 'QCOW2':
details['type'] = 'qcow2'
else:
err_msg = _("VDU {0}, Currently only QCOW2 images "
"are supported in artifacts ({1}:{2})"). \
format(self.name, key, value)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
elif name == 'file':
details['file'] = value
else:
self.log.warn(_("VDU {0}, unsuported attribute {1}").
format(self.name, name))
if len(details):
arts[key] = details
else:
arts[key] = self.artifacts[key]
self.log.debug(_("VDU {0} artifacts: {1}").
format(self.name, arts))
self.artifacts = arts
def main(self, raw_args=None):
args = self._parse_args(raw_args)
if self.log is None:
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
self.log = logging.getLogger("tosca-translator")
self.template_file = args.template_file
parsed_params = {}
if args.parameters:
parsed_params = self._parse_parameters(args.parameters)
self.archive = False
if args.archive:
self.archive = True
self.tmpdir = None
if args.validate_only:
a_file = os.path.isfile(args.template_file)
tpl = ToscaTemplate(self.template_file, parsed_params, a_file)
self.log.debug(_('Template = {}').format(tpl.__dict__))
msg = (_('The input {} successfully passed ' \
'validation.').format(self.template_file))
print(msg)
else:
self.use_gi = not args.no_gi
tpl = self._translate("tosca", parsed_params)
if tpl:
return self._write_output(tpl, args.output_dir)
def handle_requirements(self, nodes):
tosca_reqs = self.get_tosca_reqs()
self.log.debug("VNF {0} requirements: {1}".
format(self.name, tosca_reqs))
try:
for req in tosca_reqs:
if 'vdus' in req:
target = req['vdus']['target']
node = self.get_node_with_name(target, nodes)
if node:
self._vdus.append(node)
node._vnf = self
# Add the VDU id to mgmt-intf
if 'mgmt-interface' in self.properties:
self.properties['mgmt-interface']['vdu-id'] = \
node.id
else:
err_msg = _("VNF {0}, VDU {1} specified not found"). \
format(self.name, target)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
except Exception as e:
err_msg = _("Exception getting VDUs for VNF {0}: {1}"). \
format(self.name, e)
self.log.error(err_msg)
raise e
self.log.debug(_("VNF {0} properties: {1}").
format(self.name, self.properties))
def _create_archive(self, desc_id, output_dir):
"""Create a tar.gz archive for the descriptor"""
aname = desc_id + '.tar.gz'
apath = os.path.join(output_dir, aname)
self.log.debug(_("Generating archive: {}").format(apath))
prevdir = os.getcwd()
os.chdir(output_dir)
# Generate the archive
tar_cmd = "tar zcvf {} {}".format(apath, desc_id)
self.log.debug(_("Generate archive: {}").format(tar_cmd))
try:
subprocess.check_call(tar_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
return apath
except subprocess.CalledProcessError as e:
msg = _("Error creating archive with {}: {}"). \
format(tar_cmd, e)
self.log.error(msg)
raise ToscaCreateArchiveError(msg)
finally:
os.chdir(prevdir)
def main(self, raw_args=None):
args = self._parse_args(raw_args)
if self.log is None:
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
self.log = logging.getLogger("tosca-translator")
self.template_file = args.template_file
parsed_params = {}
if args.parameters:
parsed_params = self._parse_parameters(args.parameters)
self.archive = False
if args.archive:
self.archive = True
self.tmpdir = None
if args.validate_only:
a_file = os.path.isfile(args.template_file)
tpl = ToscaTemplate(self.template_file, parsed_params, a_file)
self.log.debug(_('Template = {}').format(tpl.__dict__))
msg = (_('The input {} successfully passed ' \
'validation.').format(self.template_file))
print(msg)
else:
self.use_gi = not args.no_gi
tpl = self._translate("tosca", parsed_params)
if tpl:
return self._write_output(tpl, args.output_dir)
def validate_properties(self, properties, required=None, optional=None):
if not isinstance(properties, dict):
err_msg = _("Properties for {0}({1}) is not right type"). \
format(self.name, self.type_)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
if required:
# Check if the required properties are present
if not set(required).issubset(properties.keys()):
for key in required:
if key not in properties:
err_msg = _("Property {0} is not defined "
"for {1}({2})"). \
format(key, self.name, self.type_)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
# Check for unknown properties
for key in properties.keys():
if (key not in required or
key not in optional):
self.log.warn(_("Property {0} not supported for {1}({2}), "
"will be ignored.").
format(key, self.name, self.type_))
def _translate(self, sourcetype, parsed_params):
output = None
# Check the input file
path = os.path.abspath(self.template_file)
self.in_file = path
a_file = os.path.isfile(path)
if not a_file:
msg = _("The path {} is not a valid file."). \
format(self.template_file)
self.log.error(msg)
raise ValueError(msg)
# Get the file type
self.ftype = self._get_file_type()
self.log.debug(_("Input file {0} is of type {1}").
format(path, self.ftype))
if sourcetype == "tosca":
entry_file = self.get_entry_file()
if entry_file:
self.log.debug(_('Loading the tosca template.'))
tosca = ToscaTemplate(entry_file, parsed_params, True)
self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__))
translator = TOSCATranslator(self.log, tosca, parsed_params,
use_gi=self.use_gi)
self.log.debug(_('Translating the tosca template.'))
output = translator.translate()
return output
def translate(self,
template_file,
output_dir=None,
use_gi=True,
archive=False,):
self.template_file = template_file
# Check the input file
path = os.path.abspath(template_file)
self.in_file = path
a_file = os.path.isfile(path)
if not a_file:
msg = _("The path {0} is not a valid file.").format(template_file)
self.log.error(msg)
raise ValueError(msg)
# Get the file type
self.ftype = self._get_file_type()
self.log.debug(_("Input file {0} is of type {1}").
format(path, self.ftype))
self.archive = archive
self.tmpdir = None
self.use_gi = use_gi
tpl = self._translate("tosca", {})
if tpl:
return self._write_output(tpl, output_dir)
def handle_requirements(self, nodes):
tosca_reqs = self.get_tosca_reqs()
self.log.debug("VNF {0} requirements: {1}".format(
self.name, tosca_reqs))
try:
for req in tosca_reqs:
if 'vdus' in req:
target = req['vdus']['target']
node = self.get_node_with_name(target, nodes)
if node:
self._vdus.append(node)
node._vnf = self
# Add the VDU id to mgmt-intf
if 'mgmt-interface' in self.properties:
self.properties['mgmt-interface']['vdu-id'] = \
node.id
if 'vdu' in self.properties['mgmt-interface']:
# Older yang
self.properties['mgmt-interface'].pop('vdu')
else:
err_msg = _("VNF {0}, VDU {1} specified not found"). \
format(self.name, target)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
except Exception as e:
err_msg = _("Exception getting VDUs for VNF {0}: {1}"). \
format(self.name, e)
self.log.error(err_msg)
raise e
self.log.debug(
_("VNF {0} properties: {1}").format(self.name, self.properties))
def validate_properties(self, properties, required=None, optional=None):
if not isinstance(properties, dict):
err_msg = _("Properties for {0}({1}) is not right type"). \
format(self.name, self.type_)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
if required:
# Check if the required properties are present
if not set(required).issubset(properties.keys()):
for key in required:
if key not in properties:
err_msg = _("Property {0} is not defined "
"for {1}({2})"). \
format(key, self.name, self.type_)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
# Check for unknown properties
for key in properties.keys():
if (key not in required or
key not in optional):
self.log.warn(_("Property {0} not supported for {1}({2}), "
"will be ignored.").
format(key, self.name, self.type_))
def translate(
self,
template_file,
output_dir=None,
use_gi=True,
archive=False,
):
self.template_file = template_file
# Check the input file
path = os.path.abspath(template_file)
self.in_file = path
a_file = os.path.isfile(path)
if not a_file:
msg = _("The path {0} is not a valid file.").format(template_file)
self.log.error(msg)
raise ValueError(msg)
# Get the file type
self.ftype = self._get_file_type()
self.log.debug(
_("Input file {0} is of type {1}").format(path, self.ftype))
self.archive = archive
self.tmpdir = None
self.use_gi = use_gi
tpl = self._translate("tosca", {})
if tpl:
return self._write_output(tpl, output_dir)
def handle_capabilities(self):
def get_vm_flavor(specs):
vm_flavor = {}
if 'num_cpus' in specs:
vm_flavor['vcpu-count'] = specs['num_cpus']
else:
vm_flavor['vcpu-count'] = 1
if 'mem_size' in specs:
vm_flavor['memory-mb'] = (ScalarUnit_Size(
specs['mem_size']).get_num_from_scalar_unit('MB'))
else:
vm_flavor['memory-mb'] = 512
if 'disk_size' in specs:
vm_flavor['storage-gb'] = (ScalarUnit_Size(
specs['disk_size']).get_num_from_scalar_unit('GB'))
else:
vm_flavor['storage-gb'] = 4
return vm_flavor
tosca_caps = self.get_tosca_caps()
self.log.debug(
_("VDU {0} tosca capabilites: {1}").format(self.name, tosca_caps))
if 'host' in tosca_caps:
self.properties['vm-flavor'] = get_vm_flavor(tosca_caps['host'])
self.log.debug(
_("VDU {0} properties: {1}").format(self.name,
self.properties))
def handle_properties(self, nodes, groups):
tosca_props = self.get_tosca_group_props()
self.log.debug(
_("{0} with tosca properties: {1}").format(self, tosca_props))
if 'name ' in tosca_props:
self.properties['name'] = tosca_props['name']
if 'max_instance_count' in tosca_props:
self.properties['max-instance-count'] = tosca_props[
'max_instance_count']
if 'min_instance_count' in tosca_props:
self.properties['min-instance-count'] = tosca_props[
'min_instance_count']
self.properties['vnfd-member'] = []
def _get_node(name):
for node in nodes:
if node.name == name:
return node
if 'vnfd_members' in tosca_props:
for member, count in tosca_props['vnfd_members'].items():
node = _get_node(member)
if node:
memb = {}
memb['member-vnf-index-ref'] = node.get_member_vnf_index()
memb['count'] = count
self.properties['vnfd-member'].append(memb)
else:
err_msg = _("{0}: Did not find the member node {1} in "
"resources list"). \
format(self, member)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
def _validate_action(action):
for group in groups:
if group.validate_primitive(action):
return True
return False
self.properties['scaling-config-action'] = []
if 'config_actions' in tosca_props:
for action, value in tosca_props['config_actions'].items():
conf = {}
if _validate_action(value):
conf['trigger'] = action
conf['ns-service-primitive-name-ref'] = value
self.properties['scaling-config-action'].append(conf)
else:
err_msg = _("{0}: Did not find the action {1} in "
"config primitives"). \
format(self, action)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
self.log.debug(_("{0} properties: {1}").format(self, self.properties))
def _translate(self, sourcetype, path, parsed_params, a_file):
output = None
if sourcetype == "tosca":
self.log.debug(_('Loading the tosca template.'))
tosca = ToscaTemplate(path, parsed_params, a_file)
self.log.debug(_('TOSCA Template: {}').format(tosca.__dict__))
translator = TOSCATranslator(self.log, tosca, parsed_params,
use_gi=self.use_gi)
self.log.debug(_('Translating the tosca template.'))
output = translator.translate()
return output
def _write_output(self, output, output_dir=None):
if output:
for key in output.keys():
if output_dir:
# Create sub dirs like nsd, vnfd etc
subdir = os.path.join(output_dir, key)
os.makedirs(subdir, exist_ok=True)
for desc in output[key]:
if output_dir:
output_file = os.path.join(subdir,
desc['name']+'.yml')
self.log.debug(_("Writing file {0}").
format(output_file))
with open(output_file, 'w+') as f:
f.write(desc['yang'])
else:
print(_("Descriptor {0}:\n{1}").
format(desc['name'], desc['yang']))
# Copy other directories, if present in zip
if output_dir and self.ftype == self.ZIP:
self.log.debug(_("Input is zip file, copy dirs"))
# Unzip the file to a tmp location
with tempfile.TemporaryDirectory() as tmpdirname:
prevdir = os.getcwd()
try:
with zipfile.ZipFile(self.in_file) as zf:
zf.extractall(tmpdirname)
os.chdir(tmpdirname)
dirs = [d for d in os.listdir(tmpdirname)
if os.path.isdir(d)]
for d in dirs:
if d in self.COPY_DIRS:
shutil.move(d, output_dir)
except Exception as e:
msg = _("Exception extracting input file {0}: {1}"). \
format(self.in_file, e)
self.log.error(msg)
raise e
finally:
os.chdir(prevdir)
# Create checkum for all files
flist = {}
for root, dirs, files in os.walk(output_dir):
rel_dir = root.replace(output_dir+'/', '')
for f in files:
flist[os.path.join(rel_dir, f)] = \
ChecksumUtils.get_md5(os.path.join(root, f))
self.log.debug(_("Files in output_dir: {}").format(flist))
chksumfile = os.path.join(output_dir, 'checksums.txt')
with open(chksumfile, 'w') as c:
for key in sorted(flist.keys()):
c.write("{} {}\n".format(flist[key], key))
def _copy_supporting_files(self, output_dir, files):
# Copy supporting files, if present in archive
if self.tmpdir:
# The files are refered relative to the definitions directory
arc_dir = os.path.join(self.tmpdir, self.prefix, 'Definitions')
prevdir = os.getcwd()
try:
os.chdir(arc_dir)
for fn in files:
fname = fn['name']
fpath = os.path.abspath(fname)
ty = fn['type']
if ty == 'image':
dest = os.path.join(output_dir, 'images')
elif ty == 'script':
dest = os.path.join(output_dir, 'scripts')
elif ty == 'icons':
dest = os.path.join(output_dir, 'icons')
elif ty == 'cloud_init':
dest = os.path.join(output_dir, 'cloud_init')
else:
self.log.warn(
_("Unknown file type {0} for {1}").format(
ty, fname))
continue
self.log.debug(
_("File type {0} copy from {1} to {2}").format(
ty, fpath, dest))
if os.path.exists(fpath):
# Copy the files to the appropriate dir
self.log.debug(
_("Copy file(s) {0} to {1}").format(fpath, dest))
if os.path.isdir(fpath):
# Copy directory structure like charm dir
shutil.copytree(fpath, dest)
else:
# Copy a single file
os.makedirs(dest, exist_ok=True)
shutil.copy2(fpath, dest)
else:
self.log.warn(
_("Could not find file {0} at {1}").format(
fname, fpath))
except Exception as e:
self.log.error(
_("Exception copying files {0}: {1}").format(arc_dir, e))
self.log.exception(e)
finally:
os.chdir(prevdir)
def handle_properties(self, nodes, groups):
tosca_props = self.details
self.log.debug(_("{0} with tosca properties: {1}").
format(self, tosca_props))
self.properties['name'] = tosca_props['name']
self.properties['max-instance-count'] = \
tosca_props['max_instance_count']
self.properties['min-instance-count'] = \
tosca_props['min_instance_count']
self.properties['vnfd-member'] = []
def _get_node(name):
for node in nodes:
if node.name == name:
return node
for member, count in tosca_props['vnfd_members'].items():
node = _get_node(member)
if node:
memb = {}
memb['member-vnf-index-ref'] = node.get_member_vnf_index()
memb['count'] = count
self.properties['vnfd-member'].append(memb)
else:
err_msg = _("{0}: Did not find the member node {1} in "
"resources list"). \
format(self, member)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
def _validate_action(action):
for group in groups:
if group.validate_primitive(action):
return True
return False
self.properties['scaling-config-action'] = []
for action, value in tosca_props['config_actions'].items():
conf = {}
if _validate_action(value):
conf['trigger'] = action
conf['ns-config-primitive-name-ref'] = value
self.properties['scaling-config-action'].append(conf)
else:
err_msg = _("{0}: Did not find the action {1} in "
"config primitives"). \
format(self, action)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
self.log.debug(_("{0} properties: {1}").format(self, self.properties))
def get_property(self, args):
# TODO(Philip): Should figure out how to get this resolved
# by tosca-parser using GetProperty
if isinstance(args, list):
if len(args) == 2 and \
args[0] == 'SELF':
if args[1] in self.properties:
return self.properties[args[1]]
else:
self.log.error(_("{0}, property {} not defined").
format(self.name, args[1]))
return
self.log.error(_("Get property for {0} of type {1} not supported").
format(self.name, args))
def get_property(self, args):
# TODO(Philip): Should figure out how to get this resolved
# by tosca-parser using GetProperty
if isinstance(args, list):
if len(args) == 2 and \
args[0] == 'SELF':
if args[1] in self.properties:
return self.properties[args[1]]
else:
self.log.error(_("{0}, property {} not defined").
format(self.name, args[1]))
return
self.log.error(_("Get property for {0} of type {1} not supported").
format(self.name, args))
def _copy_supporting_files(self, output_dir, files):
# Copy supporting files, if present in archive
if self.tmpdir:
# The files are refered relative to the definitions directory
arc_dir = os.path.join(self.tmpdir,
self.prefix,
'Definitions')
prevdir = os.getcwd()
try:
os.chdir(arc_dir)
for fn in files:
fname = fn['name']
fpath = os.path.abspath(fname)
ty = fn['type']
if ty == 'image':
dest = os.path.join(output_dir, 'images')
elif ty == 'script':
dest = os.path.join(output_dir, 'scripts')
else:
self.log.warn(_("Unknown file type {0} for {1}").
format(ty, fname))
continue
self.log.debug(_("File type {0} copy from {1} to {2}").
format(ty, fpath, dest))
if os.path.exists(fpath):
# Copy the files to the appropriate dir
self.log.debug(_("Copy file(s) {0} to {1}").
format(fpath, dest))
if os.path.isdir(fpath):
# Copy directory structure like charm dir
shutil.copytree(fpath, dest)
else:
# Copy a single file
os.makedirs(dest, exist_ok=True)
shutil.copy2(fpath, dest)
else:
self.log.warn(_("Could not find file {0} at {1}").
format(fname, fpath))
except Exception as e:
self.log.error(_("Exception copying files {0}: {1}").
format(arc_dir, e))
self.log.exception(e)
finally:
os.chdir(prevdir)
def handle_requirements(self, nodes, policies, vnf_type_to_vdus_map):
tosca_reqs = self.get_tosca_reqs()
for req in tosca_reqs:
for key, value in req.items():
if 'target' in value:
self._reqs[key] = value['target']
for policy in policies:
if hasattr(policy, '_vnf_name') and policy._vnf_name == self.name:
self._policies.append(policy)
if self.vnf_type in vnf_type_to_vdus_map:
for vdu_node_name in vnf_type_to_vdus_map[self.vnf_type]:
node = self.get_node_with_name(vdu_node_name, nodes)
if node:
self._vdus.append(node)
node._vnf = self
# Add the VDU id to mgmt-intf
if 'mgmt-interface' in self.properties:
self.properties['mgmt-interface']['vdu-id'] = \
node.id
if 'vdu' in self.properties['mgmt-interface']:
# Older yang
self.properties['mgmt-interface'].pop('vdu')
else:
err_msg = _("VNF {0}, VDU {1} specified not found"). \
format(self.name, target)
self.log.error(err_msg)
raise ValidationError(message=err_msg)
def update_image_checksum(self, in_file):
# Create image checksum
# in_file is the TOSCA yaml file location
if self._image is None:
return
self.log.debug("Update image: {}".format(in_file))
if os.path.exists(in_file):
in_dir = os.path.dirname(in_file)
img_dir = os.path.dirname(self._image)
abs_dir = os.path.normpath(
os.path.join(in_dir, img_dir))
self.log.debug("Abs path: {}".format(abs_dir))
if os.path.isdir(abs_dir):
img_path = os.path.join(abs_dir,
os.path.basename(self._image))
self.log.debug(_("Image path: {0}").
format(img_path))
if os.path.exists(img_path):
# TODO (pjoseph): To be fixed when we can retrieve
# the VNF image in Launchpad.
# Check if the file is not size 0
# else it is a dummy file and to be ignored
if os.path.getsize(img_path) != 0:
self._image_cksum = ChecksumUtils.get_md5(img_path,
log=self.log)
def update_image_checksum(self, in_file):
# Create image checksum
# in_file is the TOSCA yaml file location
if self._image is None:
return
self.log.debug("Update image: {}".format(in_file))
if os.path.exists(in_file):
in_dir = os.path.dirname(in_file)
img_dir = os.path.dirname(self._image)
abs_dir = os.path.normpath(
os.path.join(in_dir, img_dir))
self.log.debug("Abs path: {}".format(abs_dir))
if os.path.isdir(abs_dir):
img_path = os.path.join(abs_dir,
os.path.basename(self._image))
self.log.debug(_("Image path: {0}").
format(img_path))
if os.path.exists(img_path):
# TODO (pjoseph): To be fixed when we can retrieve
# the VNF image in Launchpad.
# Check if the file is not size 0
# else it is a dummy file and to be ignored
if os.path.getsize(img_path) != 0:
self._image_cksum = ChecksumUtils.get_md5(img_path,
log=self.log)
def generate_yang_model(self, nsd, vnfds, use_gi=False):
"""Generate yang model for the node"""
self.log.debug(_("Generate YANG model for {0}").
format(self))
for key in ToscaNfvVnf.IGNORE_PROPS:
if key in self.properties:
self.properties.pop(key)
if use_gi:
return self.generate_yang_model_gi(nsd, vnfds)
vnfd = {}
vnfd.update(self.properties)
# Update vnf configuration on mgmt interface
vnfd['mgmt-interface']['vnf-configuration'] = self._vnf_config
# Update the VDU properties
vnfd['vdu'] = []
for vdu in self._vdus:
vnfd['vdu'].append(vdu.generate_yang_submodel())
vnfds.append(vnfd)
# Update constituent vnfd in nsd
if 'constituent-vnfd' not in nsd:
nsd['constituent-vnfd'] = []
nsd['constituent-vnfd'].append(self._const_vnfd)
def _generate_type_map(log):
'''Generate TOSCA translation types map.
Load user defined classes from location path specified in conf file.
Base classes are located within the tosca directory.
'''
# Base types directory
BASE_PATH = 'rift/mano/tosca_translator/rwmano/tosca'
# Custom types directory defined in conf file
custom_path = translatorConfig.get_value('DEFAULT',
'custom_types_location')
# First need to load the parent module, for example 'contrib.mano',
# for all of the dynamically loaded classes.
classes = []
TranslateNodeTemplates._load_classes(log, (BASE_PATH, custom_path),
classes)
try:
types_map = {clazz.toscatype: clazz for clazz in classes}
log.debug(_("Type maps loaded: {}").format(types_map.keys()))
except AttributeError as e:
raise ToscaClassAttributeError(message=e.message)
return types_map
def vnf(self, vnf):
if self._vnf:
err_msg = (_('VDU {0} already has a VNF {1} associated').
format(self, self._vnf))
self.log.error(err_msg)
raise ValidationError(message=err_msg)
self._vnf = vnf
def vnf(self, vnf):
if self._vnf:
err_msg = (_('VDU {0} already has a VNF {1} associated').format(
self, self._vnf))
self.log.error(err_msg)
raise ValidationError(message=err_msg)
self._vnf = vnf
def _generate_type_map(log):
'''Generate TOSCA translation types map.
Load user defined classes from location path specified in conf file.
Base classes are located within the tosca directory.
'''
# Base types directory
BASE_PATH = 'rift/mano/tosca_translator/rwmano/tosca'
# Custom types directory defined in conf file
custom_path = translatorConfig.get_value('DEFAULT',
'custom_types_location')
# First need to load the parent module, for example 'contrib.mano',
# for all of the dynamically loaded classes.
classes = []
TranslateNodeTemplates._load_classes(log,
(BASE_PATH, custom_path),
classes)
try:
types_map = {clazz.toscatype: clazz for clazz in classes}
log.debug(_("Type maps loaded: {}").format(types_map.keys()))
except AttributeError as e:
raise ToscaClassAttributeError(message=e.message)
return types_map
def generate_yang_model(self, nsd, vnfds, use_gi=False):
"""Generate yang model for the node"""
self.log.debug(_("Generate YANG model for {0}").format(self))
for key in ToscaNfvVnf.IGNORE_PROPS:
if key in self.properties:
self.properties.pop(key)
if use_gi:
return self.generate_yang_model_gi(nsd, vnfds)
vnfd = {}
vnfd.update(self.properties)
# Update vnf configuration on mgmt interface
vnfd['mgmt-interface']['vnf-configuration'] = self._vnf_config
# Update the VDU properties
vnfd['vdu'] = []
for vdu in self._vdus:
vnfd['vdu'].append(vdu.generate_yang_submodel())
vnfds.append(vnfd)
# Update constituent vnfd in nsd
if 'constituent-vnfd' not in nsd:
nsd['constituent-vnfd'] = []
nsd['constituent-vnfd'].append(self._const_vnfd)
def _write_output(self, output, output_dir=None):
out_files = []
if output_dir:
output_dir = os.path.abspath(output_dir)
if output:
# Do the VNFDs first and then NSDs as later when
# loading in launchpad, VNFDs need to be loaded first
for key in [ManoTemplate.VNFD, ManoTemplate.NSD]:
for desc in output[key]:
if output_dir:
desc_id = desc[ManoTemplate.ID]
# Create separate directories for each descriptors
# Use the descriptor id to avoid name clash
subdir = os.path.join(output_dir, desc_id)
os.makedirs(subdir)
output_file = os.path.join(
subdir, desc[ManoTemplate.NAME] + '.yml')
self.log.debug(
_("Writing file {0}").format(output_file))
with open(output_file, 'w+') as f:
f.write(desc[ManoTemplate.YANG])
if ManoTemplate.FILES in desc:
self._copy_supporting_files(
subdir, desc[ManoTemplate.FILES])
if self.archive:
# Create checksum file
self._create_checksum_file(subdir)
out_files.append(
self._create_archive(desc_id, output_dir))
# Remove the desc directory
shutil.rmtree(subdir)
else:
print(
_("Descriptor {0}:\n{1}").format(
desc[ManoTemplate.NAME],
desc[ManoTemplate.YANG]))
if output_dir and self.archive:
# Return the list of archive files
return out_files
def main(self, log, args):
self.log = log
print("Args: {}".format(args))
self.log.debug(_("Args: {0}").format(args))
if args.type not in self.SUPPORTED_TYPES:
self.log.error(_("Unsupported file type {0}").format(args.type))
exit(1)
with open(args.generated_file) as g:
gen_data = g.read()
json_gen = json.loads(gen_data)
self.log.debug(_("Generated: {0}").format(json_gen))
with open(args.expected_file) as e:
exp_data = e.read()
json_exp = json.loads(exp_data)
self.log.debug(_("Expected: {0}").format(json_exp))
diff = DeepDiff(json_exp, json_gen)
self.log.debug(_("Keys in diff: {0}").format(diff.keys()))
self.log.info(_("Differences:\n"))
d = pprint.pformat(diff, indent=self.INDENT)
self.log.info("Differences:\n{0}".format(d))
if len(set(self.ERROR_ITEMS).intersection(diff.keys())):
diff_str = pprint.pformat(diff)
msg = _("Found item changes: {0}").format(diff_str)
self.log.error(msg)
raise ValueError(msg)
def main(self, raw_args=None, log=None):
args = self._parse_args(raw_args)
if log is None:
if args.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.ERROR)
log = logging.getLogger("tosca-translator")
self.log = log
path = os.path.abspath(args.template_file)
self.in_file = path
a_file = os.path.isfile(path)
if not a_file:
msg = _("The path %(path)s is not a valid file.") % {
'path': args.template_file}
log.error(msg)
raise ValueError(msg)
# Get the file type
self.ftype = self._get_file_type()
self.log.debug(_("Input file {0} is of type {1}").
format(path, self.ftype))
template_type = args.template_type
if template_type not in self.SUPPORTED_TYPES:
msg = _("%(value)s is not a valid template type.") % {
'value': template_type}
log.error(msg)
raise ValueError(msg)
parsed_params = {}
if args.parameters:
parsed_params = self._parse_parameters(args.parameters)
if template_type == 'tosca' and args.validate_only:
tpl = ToscaTemplate(path, parsed_params, a_file)
log.debug(_('Template = {}').format(tpl.__dict__))
msg = (_('The input "%(path)s" successfully passed '
'validation.') % {'path': path})
print(msg)
else:
self.use_gi = not args.no_gi
tpl = self._translate(template_type, path, parsed_params,
a_file)
if tpl:
self._write_output(tpl, args.output_dir)
def generate_yang_submodel(self):
"""Generate yang model for the VDU"""
self.log.debug(_("Generate YANG model for {0}").format(self))
self._update_properties_for_model()
vdu = self.properties
return vdu
def get_yang_model_gi(self, nsd, vnfds):
props = convert_keys_to_python(self.properties)
try:
nsd.scaling_group_descriptor.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception nsd scaling group from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def get_yang_model_gi(self, nsd, vnfds):
props = convert_keys_to_python(self.properties)
try:
nsd.initial_config_primitive.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception nsd initial config from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def generate_yang_model_gi(self, nsd, vnfds):
props = convert_keys_to_python(self.properties)
try:
nsd.vld.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception vld from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def get_yang_model_gi(self, nsd, vnfds):
props = convert_keys_to_python(self.properties)
try:
nsd.scaling_group_descriptor.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception nsd scaling group from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def _write_output(self, output, output_dir=None):
out_files = []
if output_dir:
output_dir = os.path.abspath(output_dir)
if output:
# Do the VNFDs first and then NSDs as later when
# loading in launchpad, VNFDs need to be loaded first
for key in [ManoTemplate.VNFD, ManoTemplate.NSD]:
for desc in output[key]:
if output_dir:
desc_id = desc[ManoTemplate.ID]
# Create separate directories for each descriptors
# Use the descriptor id to avoid name clash
subdir = os.path.join(output_dir, desc_id)
os.makedirs(subdir)
output_file = os.path.join(subdir,
desc[ManoTemplate.NAME]+'.yml')
self.log.debug(_("Writing file {0}").
format(output_file))
with open(output_file, 'w+') as f:
f.write(desc[ManoTemplate.YANG])
if ManoTemplate.FILES in desc:
self._copy_supporting_files(subdir,
desc[ManoTemplate.FILES])
if self.archive:
# Create checksum file
self._create_checksum_file(subdir)
out_files.append(self._create_archive(desc_id,
output_dir))
# Remove the desc directory
shutil.rmtree(subdir)
else:
print(_("Descriptor {0}:\n{1}").
format(desc[ManoTemplate.NAME],
desc[ManoTemplate.YANG]))
if output_dir and self.archive:
# Return the list of archive files
return out_files
def handle_properties(self, nodes, groups):
tosca_props = self.get_policy_props()
self.log.debug(_("{0} with tosca properties: {1}").
format(self, tosca_props))
self.properties['name'] = tosca_props['name']
self.properties['seq'] = int(tosca_props['seq'])
self.properties['user-defined-script'] = \
tosca_props['user_defined_script']
self.scripts.append('../scripts/{}'. \
format(tosca_props['user_defined_script']))
if 'parameter' in tosca_props:
self.properties['parameter'] = []
for parameter in tosca_props['parameter']:
self.properties['parameter'].append({
'name': parameter['name'],
'value': str(parameter['value']),
})
self.log.debug(_("{0} properties: {1}").format(self, self.properties))
def generate_yang_submodel(self):
"""Generate yang model for the VDU"""
self.log.debug(_("Generate YANG model for {0}").
format(self))
self._update_properties_for_model()
vdu = self.properties
return vdu
def __init__(self, log, tosca, parsed_params, deploy=None, use_gi=False):
super(TOSCATranslator, self).__init__()
self.log = log
self.tosca = tosca
self.mano_template = ManoTemplate(log)
self.parsed_params = parsed_params
self.deploy = deploy
self.use_gi = use_gi
self.node_translator = None
log.info(_('Initialized parmaters for translation.'))
def _load_config(cls, conf_file):
'''Private method only to be called once from the __init__ module'''
cls._translator_config = configparser.ConfigParser()
try:
cls._translator_config.read(conf_file)
except configparser.ParsingError:
msg = _('Unable to parse translator.conf file.'
'Check to see that it exists in the conf directory.')
raise exception.ConfFileParseError(message=msg)
def __init__(self, log, tosca, parsed_params, deploy=None, use_gi=False):
super(TOSCATranslator, self).__init__()
self.log = log
self.tosca = tosca
self.mano_template = ManoTemplate(log)
self.parsed_params = parsed_params
self.deploy = deploy
self.use_gi = use_gi
self.node_translator = None
log.info(_('Initialized parmaters for translation.'))
def generate_yang_submodel_gi(self, vnfd):
if vnfd is None:
return None
try:
props = convert_keys_to_python(self.properties)
vnfd.vnf_configuration.from_dict(props)
except Exception as e:
err_msg = _("{0} Exception vdu from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def handle_properties(self):
tosca_props = self.get_tosca_props()
self.log.debug(_("VDU {0} tosca properties: {1}").
format(self.name, tosca_props))
vdu_props = {}
for key, value in tosca_props.items():
vdu_props[key] = value
if 'name' not in vdu_props:
vdu_props['name'] = self.name
if 'id' not in vdu_props:
vdu_props['id'] = self.id
if 'count' not in vdu_props:
vdu_props['count'] = 1
self.log.debug(_("VDU {0} properties: {1}").
format(self.name, vdu_props))
self.properties = vdu_props
def handle_properties(self):
tosca_props = self.get_tosca_props()
if 'cidr' in tosca_props.keys():
self.log.warn(_("Support for subnet not yet "
"available. Ignoring it"))
net_props = {}
for key, value in tosca_props.items():
if key in self.NETWORK_PROPS:
if key == 'network_name':
net_props['name'] = value
elif key == 'network_id':
net_props['id'] = value
else:
net_props[key] = value
net_props['type'] = self.get_type()
if 'name' not in net_props:
# Use the node name as network name
net_props['name'] = self.name
if 'short_name' not in net_props:
# Use the node name as network name
net_props['short-name'] = self.name
if 'id' not in net_props:
net_props['id'] = self.id
if 'description' not in net_props:
net_props['description'] = self.description
if 'vendor' not in net_props:
net_props['vendor'] = self.vendor
if 'version' not in net_props:
net_props['version'] = self.version
self.log.debug(_("Network {0} properties: {1}").
format(self.name, net_props))
self.properties = net_props
def generate_yang_model_gi(self, nsd, vnfds):
vnfd_cat = RwVnfdYang.YangData_Vnfd_VnfdCatalog()
vnfd = vnfd_cat.vnfd.add()
props = convert_keys_to_python(self.properties)
for key in ToscaNfvVnf.IGNORE_PROPS:
if key in props:
props.pop(key)
try:
vnfd.from_dict(props)
except Exception as e:
err_msg = _("{0} Exception updating vnfd from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
vnfds.append(vnfd_cat)
# Update the VDU properties
for vdu in self._vdus:
vdu.generate_yang_submodel_gi(vnfd)
for policy in self._policies:
policy.generate_yang_submodel_gi(vnfd)
# Update constituent vnfd in nsd
try:
props = convert_keys_to_python(self._const_vnfd)
nsd.constituent_vnfd.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception constituent vnfd from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
# Update the vnf configuration info in mgmt_interface
props = convert_keys_to_python(self._vnf_config)
try:
vnfd.vnf_configuration.from_dict(props)
except Exception as e:
err_msg = _("{0} Exception vnfd mgmt intf from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def generate_yang_submodel_gi(self, vnfd):
if vnfd is None:
return None
self._update_properties_for_model()
props = convert_keys_to_python(self.properties)
try:
vnfd.vdu.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception vdu from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def handle_properties(self, nodes, groups):
tosca_props = self.details
self.log.debug(_("{0} with tosca properties: {1}").
format(self, tosca_props))
self.properties['name'] = tosca_props['name']
self.properties['seq'] = \
tosca_props['seq']
self.properties['user-defined-script'] = \
tosca_props['user_defined_script']
self.scripts.append('../scripts/{}'. \
format(tosca_props['user_defined_script']))
if 'parameter' in tosca_props:
self.properties['parameter'] = []
for name, value in tosca_props['parameter'].items():
self.properties['parameter'].append({
'name': name,
'value': value,
})
self.log.debug(_("{0} properties: {1}").format(self, self.properties))
def generate_yang_submodel_gi(self, vnfd):
if vnfd is None:
return None
self._update_properties_for_model()
props = convert_keys_to_python(self.properties)
try:
vnfd.vdu.add().from_dict(props)
except Exception as e:
err_msg = _("{0} Exception vdu from dict {1}: {2}"). \
format(self, props, e)
self.log.error(err_msg)
raise e
def main(self, log, args):
self.log = log
print("Args: {}".format(args))
self.log.debug(_("Args: {0}").format(args))
if args.type not in self.SUPPORTED_TYPES:
self.log.error(_("Unsupported file type {0}").
format(args.type))
exit(1)
with open(args.generated_file) as g:
gen_data = g.read()
json_gen = json.loads(gen_data)
self.log.debug(_("Generated: {0}").format(json_gen))
with open(args.expected_file) as e:
exp_data = e.read()
json_exp = json.loads(exp_data)
self.log.debug(_("Expected: {0}").format(json_exp))
diff = DeepDiff(json_exp, json_gen)
self.log.debug(_("Keys in diff: {0}").format(diff.keys()))
self.log.info(_("Differences:\n"))
d = pprint.pformat(diff, indent=self.INDENT)
self.log.info("Differences:\n{0}".format(d))
if len(set(self.ERROR_ITEMS).intersection(diff.keys())):
diff_str = pprint.pformat(diff)
msg = _("Found item changes: {0}").format(diff_str)
self.log.error(msg)
raise ValueError(msg)
