def __pce_img_gen_fc__(self, img_format, vnf, f, root, dir_o=''):
fd_path = os.path.join("{}_files".format(img_format), vnf, dir_o)
fd_path = os.path.join(self._workdir, fd_path)
os.makedirs(fd_path, exist_ok=True)
fd = os.path.join(fd_path, f)
shutil.copyfile(os.path.join(root, f), fd)
return generate_hash(fd)
def __pce_img_gen_fc__(self, img_format, vnf, f, root, dir_o=''):
fd_path = os.path.join("{}_files".format(img_format), vnf, dir_o)
fd_path = os.path.join(self._workdir, fd_path)
os.makedirs(fd_path, exist_ok=True)
fd = os.path.join(fd_path, f)
shutil.copyfile(os.path.join(root, f), fd)
return generate_hash(fd)
def generate_custom_vnfds(self):
"""
Compile information for the function descriptors, when creating a
custom package.
"""
log.info("Packaging VNF descriptors...")
for vnfd_filename in self._functions:
if not self._validator.validate_function(vnfd_filename):
log.error(
"Failed to package function '{}'".format(vnfd_filename))
return
# Create FD location
sd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy function descriptors and generate their entry points
pce = []
for vnfd_filename in self._functions:
vnfd_basename = os.path.basename(vnfd_filename)
sd = os.path.join(sd_path, vnfd_basename)
self.copy_descriptor_file(vnfd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.function_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def generate_custom_vnfds(self):
"""
Compile information for the function descriptors, when creating a
custom package.
"""
log.info("Packaging VNF descriptors...")
for vnfd_filename in self._functions:
if not self._validator.validate_function(vnfd_filename):
log.error("Failed to package function '{}'"
.format(vnfd_filename))
return
# Create FD location
sd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy function descriptors and generate their entry points
pce = []
for vnfd_filename in self._functions:
vnfd_basename = os.path.basename(vnfd_filename)
sd = os.path.join(sd_path, vnfd_basename)
self.copy_descriptor_file(vnfd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.function_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def generate_package(self, name):
"""
Generate the final package version.
:param name: The name of the final version of the package,
the project name will be used if no name provided
"""
# Validate all needed information
if not self._package_descriptor:
log.critical("Missing package descriptor. "
"Failed to generate package.")
exit(1)
if not name:
name = self._package_descriptor['vendor'] + "." + \
self._package_descriptor['name'] + "." + \
self._package_descriptor['version']
# Generate package file
zip_name = os.path.join(self._dst_path, name + '.son')
with closing(zipfile.ZipFile(zip_name, 'w')) as pck:
for base, dirs, files in os.walk(self._dst_path):
for file_name in files:
full_path = os.path.join(base, file_name)
relative_path = \
full_path[len(self._dst_path) + len(os.sep):]
if not full_path == zip_name:
pck.write(full_path, relative_path)
package_md5 = generate_hash(zip_name)
log.info("Package generated successfully.\nFile: {}\nMD5: {}\n"
.format(os.path.abspath(zip_name), package_md5))
def _validate_package_integrity(self, package, root_dir):
"""
Validate the integrity of a package.
It will validate the entry service of the package as well as its
referenced functions.
:param package: package object
:return: True if syntax is correct, None otherwise
"""
log.info("Validating integrity of package '{0}'".format(package.id))
# load referenced service descriptor files
for f in package.descriptors:
filename = os.path.join(root_dir, strip_root(f))
log.debug("Verifying file '{0}'".format(f))
if not os.path.isfile(filename):
log.error("Referenced descriptor file '{0}' is not "
"packaged.".format(f))
return
gen_md5 = generate_hash(filename)
manif_md5 = package.md5(strip_root(f))
if manif_md5 and gen_md5 != manif_md5:
log.warning("MD5 hash of file '{0}' is not equal to the "
"defined in package descriptor:\nGen MD5:\t{1}\n"
"MANIF MD5:\t{2}"
.format(f, gen_md5, manif_md5))
# configure dpath for function referencing
self.configure(dpath=os.path.join(root_dir, 'function_descriptors'))
# finally, validate the package entry service file
entry_service_file = os.path.join(
root_dir, strip_root(package.entry_service_file))
return self.validate_service(entry_service_file)
def generate_package(self, name):
"""
Generate the final package version.
:param name: The name of the final version of the package,
the project name will be used if no name provided
"""
# Validate all needed information
if not self._package_descriptor:
log.critical("Missing package descriptor. "
"Failed to generate package.")
exit(1)
if not name:
name = self._package_descriptor['vendor'] + "." + \
self._package_descriptor['name'] + "." + \
self._package_descriptor['version']
# Generate package file
zip_name = os.path.join(self._dst_path, name + '.son')
with closing(zipfile.ZipFile(zip_name, 'w')) as pck:
for base, dirs, files in os.walk(self._dst_path):
for file_name in files:
full_path = os.path.join(base, file_name)
relative_path = \
full_path[len(self._dst_path) + len(os.sep):]
if not full_path == zip_name:
pck.write(full_path, relative_path)
package_md5 = generate_hash(zip_name)
log.info("Package generated successfully.\nFile: {}\nMD5: {}\n".format(
os.path.abspath(zip_name), package_md5))
def gen_validation_key(path):
val_hash = hashlib.md5()
# generate path hash
val_hash.update(str(generate_hash(os.path.abspath(path))).encode('utf-8'))
# validation event config must also be included
val_hash.update(
repr(sorted(EventLogger.load_eventcfg().items())).encode('utf-8'))
return val_hash.hexdigest()
def gen_validation_key(path):
val_hash = hashlib.md5()
# generate path hash
val_hash.update(str(generate_hash(os.path.abspath(path)))
.encode('utf-8'))
# validation event config must also be included
val_hash.update(repr(sorted(EventLogger.load_eventcfg().items()))
.encode('utf-8'))
return val_hash.hexdigest()
def generate_vnfd_entry(self, base_path, vnf):
"""
Compile information for a specific VNF.
The VNF descriptor is validated and added to the
package.VDU image files, referenced in the VNF
descriptor, are added to the package.
:param base_path: The path where the VNF file is located
:param vnf: The VNF reference path
:return: The package content entries.
"""
# Locate VNFD
vnfd_list = [
file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file))
and file.endswith(self._project.descriptor_extension)
]
# Validate number of Yaml files
check = len(vnfd_list)
if check == 0:
log.warning("Missing VNF descriptor file in path '{}'. "
"A descriptor with '{}' extension should be "
"in this path".format(
base_path, self._project.descriptor_extension))
return
elif check > 1:
log.warning("Multiple YAML descriptors found in '{}'. "
"Ignoring path.".format(os.path.basename(base_path)))
return
else:
with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file:
vnfd = yaml.load(_file)
vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0])
# Validate VNFD
log.debug("Validating VNF descriptor file='{}'".format(vnfd_path))
if not self._validator.validate_function(
os.path.join(base_path, vnfd_list[0])):
log.exception(
"Failed to validate VNF descriptor '{}'".format(vnfd_path))
return
# Check if this VNF exists in the ns_vnf registry.
# If does not, cancel its packaging
if not self.check_in_ns_vnf(get_vnf_id(vnfd)):
log.warning("VNF id='{}' file='{}' is not referenced in the "
"service descriptor. It will be excluded from "
"the package".format(get_vnf_id(vnfd), vnfd_path))
return
pce = []
# Create fd location
fd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(fd_path, exist_ok=True)
# Copy the descriptor file
fd = os.path.join(fd_path, vnfd_list[0])
self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd)
# Generate VNFD Entry
pce_fd = dict()
pce_fd["content-type"] = "application/sonata.function_descriptor"
pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0])
pce_fd["md5"] = generate_hash(fd)
pce.append(pce_fd)
if 'virtual_deployment_units' in vnfd:
vdu_list = [
vdu for vdu in vnfd['virtual_deployment_units']
if vdu['vm_image']
]
for vdu in vdu_list:
# vm_image can be a local File, a local Dir,
# a URL or a reference to docker image
vdu_image_path = vdu['vm_image']
if validators.url(vdu_image_path): # Check if is URL/URI.
try:
# Check if the image URL exists with a short Timeout
requests.head(vdu_image_path, timeout=1)
except (requests.Timeout, requests.ConnectionError):
log.warning("Failed to verify the "
"existence of vm_image '{}'".format(
vdu['vm_image']))
# Add image URL to artifact dependencies
self._add_artifact_dependency(
name=vnfd['name'] + '-' + vdu['id'] + '-vm_image',
vendor=vnfd['vendor'],
version=vnfd['version'],
url=vdu['vm_image'],
md5='02236f2ae558018ed14b5222ef1bd9f1')
# TODO: remote url must provide md5? This is dummy!
continue
else: # Check for URL local (e.g. file:///...)
ptokens = pathlib.Path(vdu_image_path).parts
if ptokens[0] == 'file:': # URL to local file
bd = os.path.join(base_path, ptokens[1])
else: # regular filename/path
bd = os.path.join(base_path, vdu['vm_image'])
if os.path.exists(bd): # local File or local Dir
if os.path.isfile(bd):
pce.append(
self.__pce_img_gen__(base_path,
vnf,
vdu,
vdu['vm_image'],
dir_p='',
dir_o=''))
elif os.path.isdir(bd):
for root, dirs, files in os.walk(bd):
dir_o = root[len(bd):]
dir_p = dir_o.replace(os.path.sep, "/")
for f in files:
if dir_o.startswith(os.path.sep):
dir_o = dir_o[1:]
pce.append(
self.__pce_img_gen__(root,
vnf,
vdu,
f,
dir_p=dir_p,
dir_o=dir_o))
elif vdu['vm_image_format'] == 'docker':
log.debug("Referenced vm_image is docker '{}'".format(
vdu['vm_image']))
return pce
def generate_project_nsd(self):
"""
Compile information for the service descriptor section.
"""
base_path = os.path.join(self._project.project_root, 'sources', 'nsd')
if not os.path.isdir(base_path):
log.error("Missing NS directory '{}'".format(base_path))
return
# Ensure that only one NS descriptor exists
nsd_list = [
file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file))
and file.endswith(self._project.descriptor_extension)
]
check = len(nsd_list)
if check == 0:
log.error("Missing NS Descriptor file.")
return
elif check > 1:
log.error("Only one NS Descriptor file is allowed.")
return
else:
nsd_filename = nsd_list[0]
with open(os.path.join(base_path, nsd_filename), 'r') as _file:
nsd = yaml.load(_file)
# Validate NSD
log.debug(
"Validating Service Descriptor NSD='{}'".format(nsd_filename))
if not self._validator.validate_service(
os.path.join(base_path, nsd_filename)):
log.error("Failed to validate Service Descriptor '{}'. "
"Aborting package creation".format(nsd_filename))
return
# Cycle through VNFs and register their IDs for later dependency check
if 'network_functions' in nsd:
vnf_list = \
[vnf for vnf in nsd['network_functions'] if vnf['vnf_name']]
for vnf in vnf_list:
self.register_ns_vnf(
get_vnf_id_full(vnf['vnf_vendor'], vnf['vnf_name'],
vnf['vnf_version']))
# Create SD location
nsd = os.path.join(base_path, nsd_filename)
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptor file
sd = os.path.join(sd_path, nsd_filename)
self.copy_descriptor_file(nsd, sd)
# Generate NSD package content entry
pce = []
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
# Specify the NSD as THE entry service template of package descriptor
self._entry_service_template = pce_sd['name']
return pce
def generate_vnfd_entry(self, base_path, vnf):
"""
Compile information for a specific VNF.
The VNF descriptor is validated and added to the
package.VDU image files, referenced in the VNF
descriptor, are added to the package.
:param base_path: The path where the VNF file is located
:param vnf: The VNF reference path
:return: The package content entries.
"""
# Locate VNFD
vnfd_list = [file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file)) and
file.endswith(self._project.descriptor_extension)]
# Validate number of Yaml files
check = len(vnfd_list)
if check == 0:
log.warning("Missing VNF descriptor file in path '{}'. "
"A descriptor with '{}' extension should be "
"in this path"
.format(base_path,
self._project.descriptor_extension))
return
elif check > 1:
log.warning("Multiple YAML descriptors found in '{}'. "
"Ignoring path.".format(os.path.basename(base_path)))
return
else:
with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file:
vnfd = yaml.load(_file)
vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0])
# Validate VNFD
log.debug("Validating VNF descriptor file='{}'".format(vnfd_path))
if not self._validator.validate_function(os.path.join(base_path,
vnfd_list[0])):
log.exception("Failed to validate VNF descriptor '{}'"
.format(vnfd_path))
return
# Check if this VNF exists in the ns_vnf registry.
# If does not, cancel its packaging
if not self.check_in_ns_vnf(get_vnf_id(vnfd)):
log.warning("VNF id='{}' file='{}' is not referenced in the "
"service descriptor. It will be excluded from "
"the package"
.format(get_vnf_id(vnfd), vnfd_path))
return
pce = []
# Create fd location
fd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(fd_path, exist_ok=True)
# Copy the descriptor file
fd = os.path.join(fd_path, vnfd_list[0])
self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd)
# Generate VNFD Entry
pce_fd = dict()
pce_fd["content-type"] = "application/sonata.function_descriptor"
pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0])
pce_fd["md5"] = generate_hash(fd)
pce.append(pce_fd)
if 'virtual_deployment_units' in vnfd:
vdu_list = [vdu for vdu in vnfd['virtual_deployment_units']
if vdu['vm_image']]
for vdu in vdu_list:
# vm_image can be a local File, a local Dir,
# a URL or a reference to docker image
vdu_image_path = vdu['vm_image']
if validators.url(vdu_image_path): # Check if is URL/URI.
try:
# Check if the image URL exists with a short Timeout
requests.head(vdu_image_path, timeout=1)
except (requests.Timeout, requests.ConnectionError):
log.warning("Failed to verify the "
"existence of vm_image '{}'"
.format(vdu['vm_image']))
# Add image URL to artifact dependencies
self._add_artifact_dependency(
name=vnfd['name'] + '-' + vdu['id'] + '-vm_image',
vendor=vnfd['vendor'],
version=vnfd['version'],
url=vdu['vm_image'],
md5='02236f2ae558018ed14b5222ef1bd9f1')
# TODO: remote url must provide md5? This is dummy!
continue
else: # Check for URL local (e.g. file:///...)
ptokens = pathlib.Path(vdu_image_path).parts
if ptokens[0] == 'file:': # URL to local file
bd = os.path.join(base_path, ptokens[1])
else: # regular filename/path
bd = os.path.join(base_path, vdu['vm_image'])
if os.path.exists(bd): # local File or local Dir
if os.path.isfile(bd):
pce.append(self.__pce_img_gen__(
base_path, vnf, vdu, vdu['vm_image'],
dir_p='', dir_o=''))
elif os.path.isdir(bd):
for root, dirs, files in os.walk(bd):
dir_o = root[len(bd):]
dir_p = dir_o.replace(os.path.sep, "/")
for f in files:
if dir_o.startswith(os.path.sep):
dir_o = dir_o[1:]
pce.append(self.__pce_img_gen__(
root, vnf, vdu, f,
dir_p=dir_p, dir_o=dir_o))
elif vdu['vm_image_format'] == 'docker':
log.debug("Referenced vm_image is docker '{}'"
.format(vdu['vm_image']))
return pce
def generate_project_nsd(self):
"""
Compile information for the service descriptor section.
"""
base_path = os.path.join(self._project.project_root, 'sources', 'nsd')
if not os.path.isdir(base_path):
log.error("Missing NS directory '{}'".format(base_path))
return
# Ensure that only one NS descriptor exists
nsd_list = [file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file)) and
file.endswith(self._project.descriptor_extension)]
check = len(nsd_list)
if check == 0:
log.error("Missing NS Descriptor file.")
return
elif check > 1:
log.error("Only one NS Descriptor file is allowed.")
return
else:
nsd_filename = nsd_list[0]
with open(os.path.join(base_path, nsd_filename), 'r') as _file:
nsd = yaml.load(_file)
# Validate NSD
log.debug("Validating Service Descriptor NSD='{}'"
.format(nsd_filename))
if not self._validator.validate_service(os.path.join(base_path,
nsd_filename)):
log.error("Failed to validate Service Descriptor '{}'. "
"Aborting package creation".format(nsd_filename))
return
# Cycle through VNFs and register their IDs for later dependency check
if 'network_functions' in nsd:
vnf_list = \
[vnf for vnf in nsd['network_functions'] if vnf['vnf_name']]
for vnf in vnf_list:
self.register_ns_vnf(get_vnf_id_full(vnf['vnf_vendor'],
vnf['vnf_name'],
vnf['vnf_version']))
# Create SD location
nsd = os.path.join(base_path, nsd_filename)
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptor file
sd = os.path.join(sd_path, nsd_filename)
self.copy_descriptor_file(nsd, sd)
# Generate NSD package content entry
pce = []
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
# Specify the NSD as THE entry service template of package descriptor
self._entry_service_template = pce_sd['name']
return pce
