def __pce_img_gen_fc__(self, img_format, vnf, f, root, dir_o=''): fd_path = os.path.join("{}_files".format(img_format), vnf, dir_o) fd_path = os.path.join(self._workdir, fd_path) os.makedirs(fd_path, exist_ok=True) fd = os.path.join(fd_path, f) shutil.copyfile(os.path.join(root, f), fd) return generate_hash(fd)
def generate_custom_vnfds(self): """ Compile information for the function descriptors, when creating a custom package. """ log.info("Packaging VNF descriptors...") for vnfd_filename in self._functions: if not self._validator.validate_function(vnfd_filename): log.error( "Failed to package function '{}'".format(vnfd_filename)) return # Create FD location sd_path = os.path.join(self._workdir, "function_descriptors") os.makedirs(sd_path, exist_ok=True) # Copy function descriptors and generate their entry points pce = [] for vnfd_filename in self._functions: vnfd_basename = os.path.basename(vnfd_filename) sd = os.path.join(sd_path, vnfd_basename) self.copy_descriptor_file(vnfd_filename, sd) pce_sd = dict() pce_sd["content-type"] = "application/sonata.function_descriptor" pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename) pce_sd["md5"] = generate_hash(sd) pce.append(pce_sd) return pce
def generate_custom_vnfds(self): """ Compile information for the function descriptors, when creating a custom package. """ log.info("Packaging VNF descriptors...") for vnfd_filename in self._functions: if not self._validator.validate_function(vnfd_filename): log.error("Failed to package function '{}'" .format(vnfd_filename)) return # Create FD location sd_path = os.path.join(self._workdir, "function_descriptors") os.makedirs(sd_path, exist_ok=True) # Copy function descriptors and generate their entry points pce = [] for vnfd_filename in self._functions: vnfd_basename = os.path.basename(vnfd_filename) sd = os.path.join(sd_path, vnfd_basename) self.copy_descriptor_file(vnfd_filename, sd) pce_sd = dict() pce_sd["content-type"] = "application/sonata.function_descriptor" pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename) pce_sd["md5"] = generate_hash(sd) pce.append(pce_sd) return pce
def generate_package(self, name): """ Generate the final package version. :param name: The name of the final version of the package, the project name will be used if no name provided """ # Validate all needed information if not self._package_descriptor: log.critical("Missing package descriptor. " "Failed to generate package.") exit(1) if not name: name = self._package_descriptor['vendor'] + "." + \ self._package_descriptor['name'] + "." + \ self._package_descriptor['version'] # Generate package file zip_name = os.path.join(self._dst_path, name + '.son') with closing(zipfile.ZipFile(zip_name, 'w')) as pck: for base, dirs, files in os.walk(self._dst_path): for file_name in files: full_path = os.path.join(base, file_name) relative_path = \ full_path[len(self._dst_path) + len(os.sep):] if not full_path == zip_name: pck.write(full_path, relative_path) package_md5 = generate_hash(zip_name) log.info("Package generated successfully.\nFile: {}\nMD5: {}\n" .format(os.path.abspath(zip_name), package_md5))
def _validate_package_integrity(self, package, root_dir): """ Validate the integrity of a package. It will validate the entry service of the package as well as its referenced functions. :param package: package object :return: True if syntax is correct, None otherwise """ log.info("Validating integrity of package '{0}'".format(package.id)) # load referenced service descriptor files for f in package.descriptors: filename = os.path.join(root_dir, strip_root(f)) log.debug("Verifying file '{0}'".format(f)) if not os.path.isfile(filename): log.error("Referenced descriptor file '{0}' is not " "packaged.".format(f)) return gen_md5 = generate_hash(filename) manif_md5 = package.md5(strip_root(f)) if manif_md5 and gen_md5 != manif_md5: log.warning("MD5 hash of file '{0}' is not equal to the " "defined in package descriptor:\nGen MD5:\t{1}\n" "MANIF MD5:\t{2}" .format(f, gen_md5, manif_md5)) # configure dpath for function referencing self.configure(dpath=os.path.join(root_dir, 'function_descriptors')) # finally, validate the package entry service file entry_service_file = os.path.join( root_dir, strip_root(package.entry_service_file)) return self.validate_service(entry_service_file)
def generate_package(self, name): """ Generate the final package version. :param name: The name of the final version of the package, the project name will be used if no name provided """ # Validate all needed information if not self._package_descriptor: log.critical("Missing package descriptor. " "Failed to generate package.") exit(1) if not name: name = self._package_descriptor['vendor'] + "." + \ self._package_descriptor['name'] + "." + \ self._package_descriptor['version'] # Generate package file zip_name = os.path.join(self._dst_path, name + '.son') with closing(zipfile.ZipFile(zip_name, 'w')) as pck: for base, dirs, files in os.walk(self._dst_path): for file_name in files: full_path = os.path.join(base, file_name) relative_path = \ full_path[len(self._dst_path) + len(os.sep):] if not full_path == zip_name: pck.write(full_path, relative_path) package_md5 = generate_hash(zip_name) log.info("Package generated successfully.\nFile: {}\nMD5: {}\n".format( os.path.abspath(zip_name), package_md5))
def gen_validation_key(path): val_hash = hashlib.md5() # generate path hash val_hash.update(str(generate_hash(os.path.abspath(path))).encode('utf-8')) # validation event config must also be included val_hash.update( repr(sorted(EventLogger.load_eventcfg().items())).encode('utf-8')) return val_hash.hexdigest()
def gen_validation_key(path): val_hash = hashlib.md5() # generate path hash val_hash.update(str(generate_hash(os.path.abspath(path))) .encode('utf-8')) # validation event config must also be included val_hash.update(repr(sorted(EventLogger.load_eventcfg().items())) .encode('utf-8')) return val_hash.hexdigest()
def generate_vnfd_entry(self, base_path, vnf): """ Compile information for a specific VNF. The VNF descriptor is validated and added to the package.VDU image files, referenced in the VNF descriptor, are added to the package. :param base_path: The path where the VNF file is located :param vnf: The VNF reference path :return: The package content entries. """ # Locate VNFD vnfd_list = [ file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file.endswith(self._project.descriptor_extension) ] # Validate number of Yaml files check = len(vnfd_list) if check == 0: log.warning("Missing VNF descriptor file in path '{}'. " "A descriptor with '{}' extension should be " "in this path".format( base_path, self._project.descriptor_extension)) return elif check > 1: log.warning("Multiple YAML descriptors found in '{}'. " "Ignoring path.".format(os.path.basename(base_path))) return else: with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file: vnfd = yaml.load(_file) vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0]) # Validate VNFD log.debug("Validating VNF descriptor file='{}'".format(vnfd_path)) if not self._validator.validate_function( os.path.join(base_path, vnfd_list[0])): log.exception( "Failed to validate VNF descriptor '{}'".format(vnfd_path)) return # Check if this VNF exists in the ns_vnf registry. # If does not, cancel its packaging if not self.check_in_ns_vnf(get_vnf_id(vnfd)): log.warning("VNF id='{}' file='{}' is not referenced in the " "service descriptor. It will be excluded from " "the package".format(get_vnf_id(vnfd), vnfd_path)) return pce = [] # Create fd location fd_path = os.path.join(self._workdir, "function_descriptors") os.makedirs(fd_path, exist_ok=True) # Copy the descriptor file fd = os.path.join(fd_path, vnfd_list[0]) self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd) # Generate VNFD Entry pce_fd = dict() pce_fd["content-type"] = "application/sonata.function_descriptor" pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0]) pce_fd["md5"] = generate_hash(fd) pce.append(pce_fd) if 'virtual_deployment_units' in vnfd: vdu_list = [ vdu for vdu in vnfd['virtual_deployment_units'] if vdu['vm_image'] ] for vdu in vdu_list: # vm_image can be a local File, a local Dir, # a URL or a reference to docker image vdu_image_path = vdu['vm_image'] if validators.url(vdu_image_path): # Check if is URL/URI. try: # Check if the image URL exists with a short Timeout requests.head(vdu_image_path, timeout=1) except (requests.Timeout, requests.ConnectionError): log.warning("Failed to verify the " "existence of vm_image '{}'".format( vdu['vm_image'])) # Add image URL to artifact dependencies self._add_artifact_dependency( name=vnfd['name'] + '-' + vdu['id'] + '-vm_image', vendor=vnfd['vendor'], version=vnfd['version'], url=vdu['vm_image'], md5='02236f2ae558018ed14b5222ef1bd9f1') # TODO: remote url must provide md5? This is dummy! continue else: # Check for URL local (e.g. file:///...) ptokens = pathlib.Path(vdu_image_path).parts if ptokens[0] == 'file:': # URL to local file bd = os.path.join(base_path, ptokens[1]) else: # regular filename/path bd = os.path.join(base_path, vdu['vm_image']) if os.path.exists(bd): # local File or local Dir if os.path.isfile(bd): pce.append( self.__pce_img_gen__(base_path, vnf, vdu, vdu['vm_image'], dir_p='', dir_o='')) elif os.path.isdir(bd): for root, dirs, files in os.walk(bd): dir_o = root[len(bd):] dir_p = dir_o.replace(os.path.sep, "/") for f in files: if dir_o.startswith(os.path.sep): dir_o = dir_o[1:] pce.append( self.__pce_img_gen__(root, vnf, vdu, f, dir_p=dir_p, dir_o=dir_o)) elif vdu['vm_image_format'] == 'docker': log.debug("Referenced vm_image is docker '{}'".format( vdu['vm_image'])) return pce
def generate_project_nsd(self): """ Compile information for the service descriptor section. """ base_path = os.path.join(self._project.project_root, 'sources', 'nsd') if not os.path.isdir(base_path): log.error("Missing NS directory '{}'".format(base_path)) return # Ensure that only one NS descriptor exists nsd_list = [ file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file.endswith(self._project.descriptor_extension) ] check = len(nsd_list) if check == 0: log.error("Missing NS Descriptor file.") return elif check > 1: log.error("Only one NS Descriptor file is allowed.") return else: nsd_filename = nsd_list[0] with open(os.path.join(base_path, nsd_filename), 'r') as _file: nsd = yaml.load(_file) # Validate NSD log.debug( "Validating Service Descriptor NSD='{}'".format(nsd_filename)) if not self._validator.validate_service( os.path.join(base_path, nsd_filename)): log.error("Failed to validate Service Descriptor '{}'. " "Aborting package creation".format(nsd_filename)) return # Cycle through VNFs and register their IDs for later dependency check if 'network_functions' in nsd: vnf_list = \ [vnf for vnf in nsd['network_functions'] if vnf['vnf_name']] for vnf in vnf_list: self.register_ns_vnf( get_vnf_id_full(vnf['vnf_vendor'], vnf['vnf_name'], vnf['vnf_version'])) # Create SD location nsd = os.path.join(base_path, nsd_filename) sd_path = os.path.join(self._workdir, "service_descriptors") os.makedirs(sd_path, exist_ok=True) # Copy service descriptor file sd = os.path.join(sd_path, nsd_filename) self.copy_descriptor_file(nsd, sd) # Generate NSD package content entry pce = [] pce_sd = dict() pce_sd["content-type"] = "application/sonata.service_descriptor" pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename) pce_sd["md5"] = generate_hash(sd) pce.append(pce_sd) # Specify the NSD as THE entry service template of package descriptor self._entry_service_template = pce_sd['name'] return pce
def generate_vnfd_entry(self, base_path, vnf): """ Compile information for a specific VNF. The VNF descriptor is validated and added to the package.VDU image files, referenced in the VNF descriptor, are added to the package. :param base_path: The path where the VNF file is located :param vnf: The VNF reference path :return: The package content entries. """ # Locate VNFD vnfd_list = [file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file.endswith(self._project.descriptor_extension)] # Validate number of Yaml files check = len(vnfd_list) if check == 0: log.warning("Missing VNF descriptor file in path '{}'. " "A descriptor with '{}' extension should be " "in this path" .format(base_path, self._project.descriptor_extension)) return elif check > 1: log.warning("Multiple YAML descriptors found in '{}'. " "Ignoring path.".format(os.path.basename(base_path))) return else: with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file: vnfd = yaml.load(_file) vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0]) # Validate VNFD log.debug("Validating VNF descriptor file='{}'".format(vnfd_path)) if not self._validator.validate_function(os.path.join(base_path, vnfd_list[0])): log.exception("Failed to validate VNF descriptor '{}'" .format(vnfd_path)) return # Check if this VNF exists in the ns_vnf registry. # If does not, cancel its packaging if not self.check_in_ns_vnf(get_vnf_id(vnfd)): log.warning("VNF id='{}' file='{}' is not referenced in the " "service descriptor. It will be excluded from " "the package" .format(get_vnf_id(vnfd), vnfd_path)) return pce = [] # Create fd location fd_path = os.path.join(self._workdir, "function_descriptors") os.makedirs(fd_path, exist_ok=True) # Copy the descriptor file fd = os.path.join(fd_path, vnfd_list[0]) self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd) # Generate VNFD Entry pce_fd = dict() pce_fd["content-type"] = "application/sonata.function_descriptor" pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0]) pce_fd["md5"] = generate_hash(fd) pce.append(pce_fd) if 'virtual_deployment_units' in vnfd: vdu_list = [vdu for vdu in vnfd['virtual_deployment_units'] if vdu['vm_image']] for vdu in vdu_list: # vm_image can be a local File, a local Dir, # a URL or a reference to docker image vdu_image_path = vdu['vm_image'] if validators.url(vdu_image_path): # Check if is URL/URI. try: # Check if the image URL exists with a short Timeout requests.head(vdu_image_path, timeout=1) except (requests.Timeout, requests.ConnectionError): log.warning("Failed to verify the " "existence of vm_image '{}'" .format(vdu['vm_image'])) # Add image URL to artifact dependencies self._add_artifact_dependency( name=vnfd['name'] + '-' + vdu['id'] + '-vm_image', vendor=vnfd['vendor'], version=vnfd['version'], url=vdu['vm_image'], md5='02236f2ae558018ed14b5222ef1bd9f1') # TODO: remote url must provide md5? This is dummy! continue else: # Check for URL local (e.g. file:///...) ptokens = pathlib.Path(vdu_image_path).parts if ptokens[0] == 'file:': # URL to local file bd = os.path.join(base_path, ptokens[1]) else: # regular filename/path bd = os.path.join(base_path, vdu['vm_image']) if os.path.exists(bd): # local File or local Dir if os.path.isfile(bd): pce.append(self.__pce_img_gen__( base_path, vnf, vdu, vdu['vm_image'], dir_p='', dir_o='')) elif os.path.isdir(bd): for root, dirs, files in os.walk(bd): dir_o = root[len(bd):] dir_p = dir_o.replace(os.path.sep, "/") for f in files: if dir_o.startswith(os.path.sep): dir_o = dir_o[1:] pce.append(self.__pce_img_gen__( root, vnf, vdu, f, dir_p=dir_p, dir_o=dir_o)) elif vdu['vm_image_format'] == 'docker': log.debug("Referenced vm_image is docker '{}'" .format(vdu['vm_image'])) return pce
def generate_project_nsd(self): """ Compile information for the service descriptor section. """ base_path = os.path.join(self._project.project_root, 'sources', 'nsd') if not os.path.isdir(base_path): log.error("Missing NS directory '{}'".format(base_path)) return # Ensure that only one NS descriptor exists nsd_list = [file for file in os.listdir(base_path) if os.path.isfile(os.path.join(base_path, file)) and file.endswith(self._project.descriptor_extension)] check = len(nsd_list) if check == 0: log.error("Missing NS Descriptor file.") return elif check > 1: log.error("Only one NS Descriptor file is allowed.") return else: nsd_filename = nsd_list[0] with open(os.path.join(base_path, nsd_filename), 'r') as _file: nsd = yaml.load(_file) # Validate NSD log.debug("Validating Service Descriptor NSD='{}'" .format(nsd_filename)) if not self._validator.validate_service(os.path.join(base_path, nsd_filename)): log.error("Failed to validate Service Descriptor '{}'. " "Aborting package creation".format(nsd_filename)) return # Cycle through VNFs and register their IDs for later dependency check if 'network_functions' in nsd: vnf_list = \ [vnf for vnf in nsd['network_functions'] if vnf['vnf_name']] for vnf in vnf_list: self.register_ns_vnf(get_vnf_id_full(vnf['vnf_vendor'], vnf['vnf_name'], vnf['vnf_version'])) # Create SD location nsd = os.path.join(base_path, nsd_filename) sd_path = os.path.join(self._workdir, "service_descriptors") os.makedirs(sd_path, exist_ok=True) # Copy service descriptor file sd = os.path.join(sd_path, nsd_filename) self.copy_descriptor_file(nsd, sd) # Generate NSD package content entry pce = [] pce_sd = dict() pce_sd["content-type"] = "application/sonata.service_descriptor" pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename) pce_sd["md5"] = generate_hash(sd) pce.append(pce_sd) # Specify the NSD as THE entry service template of package descriptor self._entry_service_template = pce_sd['name'] return pce