def test_validate_service_invalid_topology(self):
"""
Test the validation of an integrity-invalid SONATA service.
It ensures that syntax is valid.
It ensures that integrity is valid.
"""
service_path = os.path.join(SAMPLES_DIR, 'services',
'invalid_topology.yml')
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
# syntax and integrity validation -> should return OK
validator.configure(dpath=functions_path,
syntax=True,
integrity=True,
topology=False)
validator.validate_service(service_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
# syntax, integrity and topology validation -> should return ERROR(S)
validator.configure(topology=True)
validator.validate_service(service_path)
self.assertGreater(validator.error_count, 0)
def __init__(self,
workspace,
project=None,
services=None,
functions=None,
dst_path=None,
generate_pd=True,
version="1.0"):
# Assign parameters
coloredlogs.install(level=workspace.log_level)
self._version = version
self._package_descriptor = None
self._workspace = workspace
self._project = project
self._services = services
self._functions = functions
# Create a son-access client
self._access = AccessClient(self._workspace,
log_level=self._workspace.log_level)
# Create a validator
self._validator = Validator(workspace=workspace)
self._validator.configure(syntax=True, integrity=False, topology=False)
# Create a schema validator
self._schema_validator = SchemaValidator(workspace)
# Keep track of VNF packaging referenced in NS
self._ns_vnf_registry = {}
# location to write the package
self._dst_path = dst_path if dst_path else '.'
# temporary working directory
self._workdir = '.package-' + str(time.time())
# Specifies THE service template of this package
self._entry_service_template = None
# Keep a list of repositories and
# catalogue servers that this package depend on.
# This will be included in the Package Resolver Section
self._package_resolvers = []
# Keep a list of external artifact
# dependencies that this package depends up on
# This will be included in the Artifact Dependencies Section
self._artifact_dependencies = []
# States if this package is self-contained,
# i.e. if contains all its relevant artifacts
self._sealed = True
# Clear and create package specific folder
if generate_pd:
self.init_package_skeleton()
self.build_package()
def test_validate_project_invalid(self):
"""
Tests the validation of an invalid SONATA project.
"""
prj_path = os.path.join(SAMPLES_DIR, 'sample_project_invalid')
project = Project(self._workspace, prj_path)
validator = Validator(workspace=self._workspace)
validator.validate_project(project)
self.assertGreater(validator.error_count, 0)
def test_validate_project_invalid(self):
"""
Tests the validation of an invalid SONATA project.
"""
prj_path = os.path.join(SAMPLES_DIR, 'sample_project_invalid')
project = Project(self._workspace, prj_path)
validator = Validator(workspace=self._workspace)
validator.validate_project(project)
self.assertGreater(validator.error_count, 0)
def _validate_object(keypath,
path,
obj_type,
syntax,
integrity,
topology,
pkg_signature=None,
pkg_pubkey=None):
# protect against incorrect parameters
perrors = validate_parameters(obj_type, syntax, integrity, topology)
if perrors:
return perrors, 400
rid = gen_resource_key(keypath, obj_type, syntax, integrity, topology)
vid = gen_validation_key(path)
resource = get_resource(rid)
validation = get_validation(vid)
if resource and validation:
log.info("Returning cached result for '{0}'".format(vid))
update_resource_validation(rid, vid)
return validation['result']
log.info("Starting validation [type={}, path={}, flags={}"
"resource_id:={}, validation_id={}]".format(
obj_type, path, get_flags(syntax, integrity, topology), rid,
vid))
set_resource(rid, keypath, obj_type, syntax, integrity, topology)
validator = Validator()
validator.configure(syntax,
integrity,
topology,
debug=app.config['DEBUG'],
pkg_signature=pkg_signature,
pkg_pubkey=pkg_pubkey)
# remove default dpath
validator.dpath = None
val_function = getattr(validator, 'validate_' + obj_type)
result = val_function(path)
print_result(validator, result)
json_result = gen_report_result(rid, validator)
net_topology = gen_report_net_topology(validator)
net_fwgraph = gen_report_net_fwgraph(validator)
set_validation(vid,
result=json_result,
net_topology=net_topology,
net_fwgraph=net_fwgraph)
update_resource_validation(rid, vid)
return json_result
def test_validate_project_warning(self):
"""
Tests the validation of a SONATA project with warnings.
"""
prj_path = os.path.join(SAMPLES_DIR, 'sample_project_warning')
project = Project(self._workspace, prj_path)
validator = Validator(workspace=self._workspace)
validator.validate_project(project)
self.assertEqual(validator.error_count, 0)
self.assertGreater(validator.warning_count, 0)
def test_validate_project_warning(self):
"""
Tests the validation of a SONATA project with warnings.
"""
prj_path = os.path.join(SAMPLES_DIR, 'sample_project_warning')
project = Project(self._workspace, prj_path)
validator = Validator(workspace=self._workspace)
validator.validate_project(project)
self.assertEqual(validator.error_count, 0)
self.assertGreater(validator.warning_count, 0)
def test_validate_function_valid(self):
"""
Tests the validation of a valid SONATA function.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
validator.configure(syntax=True, integrity=True, topology=True)
validator.validate_function(functions_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
def test_validate_package_invalid_md5(self):
"""
Tests the validation of a SONATA package with incorrect MD5 sums
"""
self.reset_counters()
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(val.log.error.counter, 0)
self.assertEqual(val.log.warning.counter, 4)
def test_validate_package_valid(self):
"""
Tests the validation of a valid SONATA package.
"""
self.reset_counters()
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-valid.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(val.log.error.counter, 0)
self.assertEqual(val.log.warning.counter, 0)
def test_validate_service_invalid_syntax(self):
"""
Tests the validation of an syntax-invalid SONATA service.
"""
service_path = os.path.join(SAMPLES_DIR, 'services',
'invalid_syntax.yml')
validator = Validator()
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_service(service_path)
self.assertGreater(validator.error_count, 0)
def test_validate_package_valid(self):
"""
Tests the validation of a valid SONATA package.
"""
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-valid.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
def test_validate_function_invalid_syntax(self):
"""
Tests the validation of a syntax-invalid SONATA function.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions',
'invalid_syntax')
validator = Validator()
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_function(functions_path)
self.assertGreater(validator.error_count, 0)
def test_validate_project_valid(self):
"""
Tests the validation of a valid SONATA project.
"""
self.reset_counters()
prj_path = os.path.join(SAMPLES_DIR, 'sample_project_valid')
project = Project(self._workspace, prj_path)
validator = Validator(workspace=self._workspace)
validator.validate_project(project)
self.assertEqual(val.log.error.counter, 0)
self.assertEqual(val.log.warning.counter, 0)
def test_validate_service_valid(self):
"""
Tests the validation of a valid SONATA service.
"""
service_path = os.path.join(SAMPLES_DIR, 'services', 'valid.yml')
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
validator.configure(dpath=functions_path)
validator.validate_service(service_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
def validate_function():
file = request.files['function']
filepath = upload_file(file)
validator = Validator()
validator.configure(syntax=True,
integrity=False,
topology=False,
debug=app.debug)
result = validator.validate_function(filepath)
print_result(validator, result)
remove_file(filepath)
return generate_result(validator)
def test_validate_package_signature(self):
"""
Tests the package signature validation function of son-validate.
To accomplish this:
1) a private/public key pair is generated
2) a signature is created based on a private key and a package file
3) the signature validation function is called with the generated
signature and the public key
"""
# generate private/public key pair
random_generator = Random.new().read
key = RSA.generate(1024, random_generator)
self.assertTrue(key.can_encrypt())
self.assertTrue(key.can_sign())
self.assertTrue(key.has_private())
# create signature of a file
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-valid.son')
file_data = None
try:
with open(pkg_path, 'rb') as _file:
file_data = _file.read()
except IOError as err:
print("I/O error: {0}".format(err))
pkg_hash = SHA256.new(file_data).digest()
signature = str(key.sign(pkg_hash, '')[0])
pubkey = key.publickey().exportKey('DER') # export in binary encoding
# call signature validation function
validator = Validator(workspace=self._workspace)
result = validator.validate_package_signature(pkg_path,
signature,
pubkey)
# signature must be valid
self.assertTrue(result)
# call signature validation with a different file
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
result = validator.validate_package_signature(pkg_path,
signature,
pubkey)
# signature must be invalid
self.assertFalse(result)
def test_event_config_cli(self):
"""
Tests the custom event configuration meant to be used with the CLI
"""
# backup current user eventcfg (if exists)
if os.path.isfile('eventcfg.yml'):
shutil.move('eventcfg.yml', '.eventcfg.yml.original')
# load eventdict
eventdict = EventLogger.load_eventcfg()
# report unmatched file hashes as error
eventdict['evt_pd_itg_invalid_md5'] = 'error'
# report vdu image not found as error
eventdict['evt_vnfd_itg_vdu_image_not_found'] = 'error'
# write eventdict
EventLogger.dump_eventcfg(eventdict)
# perform validation test
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
# should return 1 error
self.assertEqual(validator.error_count, 2)
self.assertEqual(validator.warning_count, 0)
# report unmatched file hashes as warning
eventdict['evt_pd_itg_invalid_md5'] = 'warning'
# do not report vdu image not found
eventdict['evt_vnfd_itg_vdu_image_not_found'] = 'none'
# write eventdict
EventLogger.dump_eventcfg(eventdict)
# perform validation test
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
# should return 1 warning
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 1)
# delete temporary eventcfg
os.remove('eventcfg.yml')
# restore user eventcfg
if os.path.isfile('.eventcfg.yml.original'):
shutil.move('.eventcfg.yml.original', 'eventcfg.yml')
def _validate_object(keypath, path, obj_type, syntax, integrity, topology,
pkg_signature=None, pkg_pubkey=None):
# protect against incorrect parameters
perrors = validate_parameters(obj_type, syntax, integrity, topology)
if perrors:
return perrors, 400
rid = gen_resource_key(keypath, obj_type, syntax, integrity, topology)
vid = gen_validation_key(path)
resource = get_resource(rid)
validation = get_validation(vid)
if resource and validation:
log.info("Returning cached result for '{0}'".format(vid))
update_resource_validation(rid, vid)
return validation['result']
log.info("Starting validation [type={}, path={}, flags={}"
"resource_id:={}, validation_id={}]"
.format(obj_type, path, get_flags(syntax, integrity, topology),
rid, vid))
set_resource(rid, keypath, obj_type, syntax, integrity, topology)
validator = Validator()
validator.configure(syntax, integrity, topology, debug=app.config['DEBUG'],
pkg_signature=pkg_signature, pkg_pubkey=pkg_pubkey)
# remove default dpath
validator.dpath = None
val_function = getattr(validator, 'validate_' + obj_type)
result = val_function(path)
print_result(validator, result)
json_result = gen_report_result(rid, validator)
net_topology = gen_report_net_topology(validator)
net_fwgraph = gen_report_net_fwgraph(validator)
set_validation(vid, result=json_result, net_topology=net_topology,
net_fwgraph=net_fwgraph)
update_resource_validation(rid, vid)
return json_result
def test_validate_function_invalid_integrity(self):
"""
Tests the validation of a integrity-invalid SONATA function.
It ensures that syntax is valid.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions',
'invalid_integrity')
validator = Validator()
# syntax validation -> should return OK
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_function(functions_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
# syntax and integrity validation -> should return ERROR(S)
validator.configure(integrity=True)
validator.validate_function(functions_path)
self.assertGreater(validator.error_count, 0)
def __init__(self, workspace, project=None, services=None, functions=None,
dst_path=None, generate_pd=True, version="1.0"):
# Assign parameters
coloredlogs.install(level=workspace.log_level)
self._version = version
self._package_descriptor = None
self._workspace = workspace
self._project = project
self._services = services
self._functions = functions
# Create a son-access client
self._access = AccessClient(self._workspace,
log_level=self._workspace.log_level)
# Create a validator
self._validator = Validator(workspace=workspace)
self._validator.configure(syntax=True, integrity=False, topology=False)
# Create a schema validator
self._schema_validator = SchemaValidator(workspace)
# Keep track of VNF packaging referenced in NS
self._ns_vnf_registry = {}
# location to write the package
self._dst_path = dst_path if dst_path else '.'
# temporary working directory
self._workdir = '.package-' + str(time.time())
# Specifies THE service template of this package
self._entry_service_template = None
# Keep a list of repositories and
# catalogue servers that this package depend on.
# This will be included in the Package Resolver Section
self._package_resolvers = []
# Keep a list of external artifact
# dependencies that this package depends up on
# This will be included in the Artifact Dependencies Section
self._artifact_dependencies = []
# States if this package is self-contained,
# i.e. if contains all its relevant artifacts
self._sealed = True
# Clear and create package specific folder
if generate_pd:
self.init_package_skeleton()
self.build_package()
def validate_package():
file = request.files['package']
filepath = upload_file(file)
syntax = (False if 'syntax' not in request.form else eval(
request.form['syntax']))
integrity = (False if 'integrity' not in request.form else eval(
request.form['integrity']))
topology = (False if 'topology' not in request.form else eval(
request.form['topology']))
validator = Validator()
validator.configure(syntax=syntax,
integrity=integrity,
topology=topology,
debug=app.debug)
result = validator.validate_package(filepath)
print_result(validator, result)
remove_file(filepath)
return generate_result(validator)
def test_validate_function_invalid_integrity(self):
"""
Tests the validation of a integrity-invalid SONATA function.
It ensures that syntax is valid.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions',
'invalid_integrity')
validator = Validator()
# syntax validation -> should return OK
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_function(functions_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
# syntax and integrity validation -> should return ERROR(S)
validator.configure(integrity=True)
validator.validate_function(functions_path)
self.assertGreater(validator.error_count, 0)
def test_validate_package_invalid_md5(self):
"""
Tests the validation of a SONATA package with incorrect MD5 sums
"""
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
eventdict = EventLogger.load_eventcfg()
invalid_md5_config = str(eventdict['evt_pd_itg_invalid_md5']).lower()
if invalid_md5_config == 'error':
error_count = 1
warn_count = 1
elif invalid_md5_config == 'warning':
error_count = 0
warn_count = 2
elif invalid_md5_config == 'none':
error_count = 0
warn_count = 1
else:
self.fail("Invalid value of event 'evt_pd_itg_invalid_md5'")
self.assertEqual(validator.error_count, error_count)
self.assertEqual(validator.warning_count, warn_count)
def test_validate_package_invalid_md5(self):
"""
Tests the validation of a SONATA package with incorrect MD5 sums
"""
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
eventdict = EventLogger.load_eventcfg()
invalid_md5_config = str(eventdict['evt_pd_itg_invalid_md5']).lower()
if invalid_md5_config == 'error':
error_count = 1
warn_count = 1
elif invalid_md5_config == 'warning':
error_count = 0
warn_count = 2
elif invalid_md5_config == 'none':
error_count = 0
warn_count = 1
else:
self.fail("Invalid value of event 'evt_pd_itg_invalid_md5'")
self.assertEqual(validator.error_count, error_count)
self.assertEqual(validator.warning_count, warn_count)
def test_validate_package_invalid_struct(self):
"""
Tests the validation of a multiple SONATA packages with a bad file
structure.
"""
# invalid struct #1
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-struct-1.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
# invalid struct #2
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-struct-2.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
def test_event_config_cli(self):
"""
Tests the custom event configuration meant to be used with the CLI
"""
# backup current user eventcfg (if exists)
if os.path.isfile('eventcfg.yml'):
shutil.move('eventcfg.yml', '.eventcfg.yml.original')
# load eventdict
eventdict = EventLogger.load_eventcfg()
# report unmatched file hashes as error
eventdict['evt_pd_itg_invalid_md5'] = 'error'
# report vdu image not found as error
eventdict['evt_vnfd_itg_vdu_image_not_found'] = 'error'
# write eventdict
EventLogger.dump_eventcfg(eventdict)
# perform validation test
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
# should return 1 error
self.assertEqual(validator.error_count, 2)
self.assertEqual(validator.warning_count, 0)
# report unmatched file hashes as warning
eventdict['evt_pd_itg_invalid_md5'] = 'warning'
# do not report vdu image not found
eventdict['evt_vnfd_itg_vdu_image_not_found'] = 'none'
# write eventdict
EventLogger.dump_eventcfg(eventdict)
# perform validation test
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
# should return 1 warning
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 1)
# delete temporary eventcfg
os.remove('eventcfg.yml')
# restore user eventcfg
if os.path.isfile('.eventcfg.yml.original'):
shutil.move('.eventcfg.yml.original', 'eventcfg.yml')
def test_validate_function_valid(self):
"""
Tests the validation of a valid SONATA function.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
validator.configure(syntax=True, integrity=True, topology=True)
validator.validate_function(functions_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
def test_validate_service_invalid_syntax(self):
"""
Tests the validation of an syntax-invalid SONATA service.
"""
service_path = os.path.join(SAMPLES_DIR, 'services',
'invalid_syntax.yml')
validator = Validator()
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_service(service_path)
self.assertGreater(validator.error_count, 0)
def test_validate_function_invalid_syntax(self):
"""
Tests the validation of a syntax-invalid SONATA function.
"""
functions_path = os.path.join(SAMPLES_DIR, 'functions',
'invalid_syntax')
validator = Validator()
validator.configure(syntax=True, integrity=False, topology=False)
validator.validate_function(functions_path)
self.assertGreater(validator.error_count, 0)
def test_validate_service_invalid_topology(self):
"""
Test the validation of an integrity-invalid SONATA service.
It ensures that syntax is valid.
It ensures that integrity is valid.
"""
service_path = os.path.join(SAMPLES_DIR, 'services',
'invalid_topology.yml')
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
# syntax and integrity validation -> should return OK
validator.configure(dpath=functions_path,
syntax=True, integrity=True, topology=False)
validator.validate_service(service_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
# syntax, integrity and topology validation -> should return ERROR(S)
validator.configure(topology=True)
validator.validate_service(service_path)
self.assertGreater(validator.error_count, 0)
def test_validate_service_valid(self):
"""
Tests the validation of a valid SONATA service.
"""
service_path = os.path.join(SAMPLES_DIR, 'services', 'valid.yml')
functions_path = os.path.join(SAMPLES_DIR, 'functions', 'valid')
validator = Validator()
validator.configure(dpath=functions_path)
validator.validate_service(service_path)
self.assertEqual(validator.error_count, 0)
self.assertEqual(validator.warning_count, 0)
def test_validate_package_invalid_integrigy(self):
"""
Tests the validation of several SONATA packages with incorrect
integrity.
"""
# invalid integrity #1
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-integrity-1.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
self.assertEqual(validator.warning_count, 0)
# invalid integrity #2
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-integrity-2.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
self.assertEqual(validator.warning_count, 0)
def test_validate_package_invalid_struct(self):
"""
Tests the validation of a multiple SONATA packages with a bad file
structure.
"""
# invalid struct #1
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-struct-1.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
# invalid struct #2
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-struct-2.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
def test_validate_package_signature(self):
"""
Tests the package signature validation function of son-validate.
To accomplish this:
1) a private/public key pair is generated
2) a signature is created based on a private key and a package file
3) the signature validation function is called with the generated
signature and the public key
"""
# generate private/public key pair
random_generator = Random.new().read
key = RSA.generate(1024, random_generator)
self.assertTrue(key.can_encrypt())
self.assertTrue(key.can_sign())
self.assertTrue(key.has_private())
# create signature of a file
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-valid.son')
file_data = None
try:
with open(pkg_path, 'rb') as _file:
file_data = _file.read()
except IOError as err:
print("I/O error: {0}".format(err))
pkg_hash = SHA256.new(file_data).digest()
signature = str(key.sign(pkg_hash, '')[0])
pubkey = key.publickey().exportKey('DER') # export in binary encoding
# call signature validation function
validator = Validator(workspace=self._workspace)
result = validator.validate_package_signature(pkg_path, signature,
pubkey)
# signature must be valid
self.assertTrue(result)
# call signature validation with a different file
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-md5.son')
validator = Validator(workspace=self._workspace)
result = validator.validate_package_signature(pkg_path, signature,
pubkey)
# signature must be invalid
self.assertFalse(result)
def test_validate_package_invalid_integrigy(self):
"""
Tests the validation of several SONATA packages with incorrect
integrity.
"""
# invalid integrity #1
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-integrity-1.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
self.assertEqual(validator.warning_count, 0)
# invalid integrity #2
pkg_path = os.path.join(SAMPLES_DIR, 'packages',
'sonata-demo-invalid-integrity-2.son')
validator = Validator(workspace=self._workspace)
validator.validate_package(pkg_path)
self.assertEqual(validator.error_count, 1)
self.assertEqual(validator.warning_count, 0)
class Packager(object):
def __init__(self,
workspace,
project=None,
services=None,
functions=None,
dst_path=None,
generate_pd=True,
version="1.0"):
# Assign parameters
coloredlogs.install(level=workspace.log_level)
self._version = version
self._package_descriptor = None
self._workspace = workspace
self._project = project
self._services = services
self._functions = functions
# Create a son-access client
self._access = AccessClient(self._workspace,
log_level=self._workspace.log_level)
# Create a validator
self._validator = Validator(workspace=workspace)
self._validator.configure(syntax=True, integrity=False, topology=False)
# Create a schema validator
self._schema_validator = SchemaValidator(workspace)
# Keep track of VNF packaging referenced in NS
self._ns_vnf_registry = {}
# location to write the package
self._dst_path = dst_path if dst_path else '.'
# temporary working directory
self._workdir = '.package-' + str(time.time())
# Specifies THE service template of this package
self._entry_service_template = None
# Keep a list of repositories and
# catalogue servers that this package depend on.
# This will be included in the Package Resolver Section
self._package_resolvers = []
# Keep a list of external artifact
# dependencies that this package depends up on
# This will be included in the Artifact Dependencies Section
self._artifact_dependencies = []
# States if this package is self-contained,
# i.e. if contains all its relevant artifacts
self._sealed = True
# Clear and create package specific folder
if generate_pd:
self.init_package_skeleton()
self.build_package()
def init_package_skeleton(self):
"""
Validate and initialize the destination folder
for the creation of the package artifacts.
"""
if os.path.isdir(self._workdir):
log.error("Internal error. Temporary workdir already exists.")
return
# workdir
os.mkdir(self._workdir)
atexit.register(shutil.rmtree, os.path.abspath(self._workdir))
# destination path
if not os.path.isdir(self._dst_path):
os.mkdir(self._dst_path)
@property
def package_descriptor(self):
return self._package_descriptor
def build_package(self):
"""
Create and set the full package descriptor as a dictionary.
It process the file by each individual section.
"""
log.info('Create Package Content Section')
package_content = self.package_pcs()
log.info('Create Package Resolver Section')
package_resolver = self.package_prs()
log.info('Create Package Dependencies Section')
package_dependencies = self.package_pds()
log.info('Create Artifact Dependencies Section')
artifact_dependencies = self.package_ads()
# The general section must be created last,
# some fields depend on prior processing
log.info('Create General Description section')
if self._project:
general_description = self.package_gds(
prj_descriptor=self._project.project_config)
else:
general_description = self.package_gds()
if not general_description:
log.error("Failed to package General Description Section.")
return
# Compile all sections in package descriptor
self._package_descriptor = general_description
if not package_content:
log.error("Failed to package Package Content Section. "
"Could not find a network service and/or its "
"referenced function descriptors")
self._package_descriptor = None
return
self._package_descriptor.update(package_content)
self._package_descriptor.update(package_resolver)
self._package_descriptor.update(package_dependencies)
self._package_descriptor.update(artifact_dependencies)
# Create the manifest folder and file
meta_inf = os.path.join(self._workdir, "META-INF")
os.makedirs(meta_inf, exist_ok=True)
with open(os.path.join(meta_inf, "MANIFEST.MF"), "w") as manifest:
manifest.write(
yaml.dump(self.package_descriptor, default_flow_style=False))
@performance
def package_gds(self, prj_descriptor=None):
"""
Compile information for the General Description Section.
This section is exclusively filled by the project descriptor
file located on the root of every project.
"""
# List of mandatory fields to be included in the GDS
gds_fields = ['vendor', 'name', 'version', 'maintainer', 'description']
gds = dict()
gds['descriptor_version'] = self._version
gds['schema'] = self._schema_validator.get_remote_schema(
SchemaValidator.SCHEMA_PACKAGE_DESCRIPTOR)
gds['sealed'] = self._sealed
if prj_descriptor:
gds['entry_service_template'] = self._entry_service_template
if 'package' not in prj_descriptor.keys():
log.error("Please define 'package' section in {}".format(
Project.__descriptor_name__))
return
errors = []
for field in gds_fields:
if field not in prj_descriptor['package'].keys():
errors.append(field)
else:
gds[field] = prj_descriptor['package'][field]
if errors:
log.error(
'Please define {} in the package section of {}'.format(
', '.join(errors), Project.__descriptor_name__))
return
else:
# TODO: what properties to set in a custom package? TBD...
gds['vendor'] = 'custom'
gds['name'] = 'package'
gds['version'] = '1.0'
gds['maintainer'] = 'developer'
gds['description'] = 'custom generated package'
return gds
@performance
def package_pcs(self):
"""
Compile information for the Package Content Section.
This section contains all the artifacts that are
contained and shipped by the package.
"""
pcs = []
# Load and add service descriptor
if self._project:
nsd = self.generate_project_nsd()
if not nsd or len(nsd) == 0:
log.error("Failed to package service descriptor")
return
pcs += nsd
elif self._services:
nsds = self.generate_custom_nsds()
if not nsds:
log.error("Failed to package service descriptors")
return
pcs += nsds
# Load and add the function descriptors
if self._project:
vnfds = self.generate_project_vnfds()
if not vnfds or len(vnfds) == 0:
log.error("Failed to package function descriptors")
return
pcs += vnfds
elif self._functions:
vnfds = self.generate_custom_vnfds()
if not vnfds:
log.error("Failed to package function descriptors")
return
pcs += vnfds
return dict(package_content=pcs)
@performance
def package_prs(self):
"""
Compile information for the Package Resolver Section.
This section contains information about catalogues
and repositories needed to resolve the dependencies
specified in this package descriptor.
"""
if len(self._package_resolvers) == 0:
log.debug("There are no required Package Resolvers. "
"This section will not be included.")
return dict()
return dict(package_resolvers=self._package_resolvers)
@performance
def package_pds(self):
"""
Compile information for the Package Dependencies
Section. This section specifies additional packages
that this package depends up on.
"""
log.debug("There are no required Package Dependencies. "
"This section will not be included.")
return dict()
@performance
def package_ads(self):
"""
Compile information for the Artifact Dependencies
Section. This section contains components that are
not included in the package but are referenced in
its descriptors. For instance, it includes the url
of vm_images used by network functions.
"""
if len(self._artifact_dependencies) == 0:
log.debug("There are no required Artifact Dependencies. "
"This section will not be included.")
return dict()
return dict(artifact_dependencies=self._artifact_dependencies)
def generate_project_nsd(self):
"""
Compile information for the service descriptor section.
"""
base_path = os.path.join(self._project.project_root, 'sources', 'nsd')
if not os.path.isdir(base_path):
log.error("Missing NS directory '{}'".format(base_path))
return
# Ensure that only one NS descriptor exists
nsd_list = [
file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file))
and file.endswith(self._project.descriptor_extension)
]
check = len(nsd_list)
if check == 0:
log.error("Missing NS Descriptor file.")
return
elif check > 1:
log.error("Only one NS Descriptor file is allowed.")
return
else:
nsd_filename = nsd_list[0]
with open(os.path.join(base_path, nsd_filename), 'r') as _file:
nsd = yaml.load(_file)
# Validate NSD
log.debug(
"Validating Service Descriptor NSD='{}'".format(nsd_filename))
if not self._validator.validate_service(
os.path.join(base_path, nsd_filename)):
log.error("Failed to validate Service Descriptor '{}'. "
"Aborting package creation".format(nsd_filename))
return
# Cycle through VNFs and register their IDs for later dependency check
if 'network_functions' in nsd:
vnf_list = \
[vnf for vnf in nsd['network_functions'] if vnf['vnf_name']]
for vnf in vnf_list:
self.register_ns_vnf(
get_vnf_id_full(vnf['vnf_vendor'], vnf['vnf_name'],
vnf['vnf_version']))
# Create SD location
nsd = os.path.join(base_path, nsd_filename)
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptor file
sd = os.path.join(sd_path, nsd_filename)
self.copy_descriptor_file(nsd, sd)
# Generate NSD package content entry
pce = []
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
# Specify the NSD as THE entry service template of package descriptor
self._entry_service_template = pce_sd['name']
return pce
def generate_custom_nsds(self):
"""
Compile information for the service descriptors, when creating a
custom package.
"""
log.info("Packaging service descriptors...")
for nsd_filename in self._services:
if not self._validator.validate_service(nsd_filename):
log.error(
"Failed to package service '{}'".format(nsd_filename))
return
# Create SD location
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptors and generate their entry points
pce = []
for nsd_filename in self._services:
nsd_basename = os.path.basename(nsd_filename)
sd = os.path.join(sd_path, nsd_basename)
self.copy_descriptor_file(nsd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def generate_project_vnfds(self):
"""
Compile information for the function descriptors, when packaging an
SDK project.
"""
# Add VNFs from project source
log.info("Packaging VNF descriptors from project source...")
pcs = self.generate_project_source_vnfds(
os.path.join(self._project.project_root, 'sources', 'vnf'))
# Verify that all VNFs from NSD were packaged
unpack_vnfs = self.get_unpackaged_ns_vnfs()
if len(unpack_vnfs) > 0:
# Load function descriptors (VNFDs) from external sources
log.info("Solving dependencies for VNF descriptors...")
if not self.load_external_vnfds(unpack_vnfs):
log.error("Unable to solve all dependencies "
"required by the service descriptor.")
return
log.info("Packaging VNF descriptors from external source...")
pcs_ext = self.generate_external_vnfds(
os.path.join(self._workspace.workspace_root,
self._workspace.vnf_catalogue_dir), unpack_vnfs)
if not pcs_ext or len(pcs_ext) == 0:
return
pcs += pcs_ext
# Verify again if all VNFs were correctly packaged
unpack_vnfs = self.get_unpackaged_ns_vnfs()
if len(unpack_vnfs) > 0:
log.error("Unable to validate all VNFs "
"required by the service descriptor.")
return
return pcs
def generate_custom_vnfds(self):
"""
Compile information for the function descriptors, when creating a
custom package.
"""
log.info("Packaging VNF descriptors...")
for vnfd_filename in self._functions:
if not self._validator.validate_function(vnfd_filename):
log.error(
"Failed to package function '{}'".format(vnfd_filename))
return
# Create FD location
sd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy function descriptors and generate their entry points
pce = []
for vnfd_filename in self._functions:
vnfd_basename = os.path.basename(vnfd_filename)
sd = os.path.join(sd_path, vnfd_basename)
self.copy_descriptor_file(vnfd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.function_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def load_external_vnfds(self, vnf_id_list):
"""
This method is responsible to load all VNFs, required
by the NS, that are not part of project source.
VNFs can be loaded from the Workspace catalog or/and
from the catalogue servers.
:param vnf_id_list: List of VNF ID to solve
:return: True for success, False for failure
"""
log.debug(
"Loading the following VNF descriptors: {}".format(vnf_id_list))
# Iterate through the VNFs required by the NS
for vnf_id in vnf_id_list:
log.debug("Probing workspace catalogue for VNF id='{}'...".format(
vnf_id))
# >> First, check if this VNF is in the workspace catalogue
catalogue_path = os.path.join(self._workspace.workspace_root,
self._workspace.vnf_catalogue_dir,
vnf_id)
if os.path.isdir(catalogue_path):
# Exists! Save catalogue path of this vnf for later packaging
log.debug(
"Found VNF id='{}' in workspace catalogue '{}'".format(
vnf_id, catalogue_path))
continue
log.debug("VNF id='{}' is not present in workspace catalogue. "
"Contacting SP Catalogue...".format(vnf_id))
# If not in WS catalogue, get the VNF from the SP Catalogues
vnfd = None
self.retrieve_external_vnf(vnf_id)
if not vnfd:
log.warning(
"VNF id='{}' is not present in SP Catalogue".format(
vnf_id))
return False
# Create dir to hold the retrieved VNF in workspace catalogue
log.debug("VNF id='{}' retrieved from the SP Catalogue. "
"Loading to workspace cache.".format(vnf_id))
os.mkdir(catalogue_path)
vnfd_f = open(
os.path.join(
catalogue_path,
vnfd['name'] + "." + self._project.descriptor_extension),
'w')
yaml.dump(vnfd, vnfd_f, default_flow_style=False)
return True
def generate_project_source_vnfds(self, base_path):
"""
Compile information for the list of VNFs
This function iterates over the different VNF entries
:param base_path: base dir location of VNF descriptors
:return:
"""
vnf_folders = filter(
lambda file: os.path.isdir(os.path.join(base_path, file)),
os.listdir(base_path))
pcs = []
for vnf in vnf_folders:
pc_entries = self.generate_vnfd_entry(os.path.join(base_path, vnf),
vnf)
if not pc_entries or len(pc_entries) == 0:
continue
for pce in iter(pc_entries):
pcs.append(pce)
return pcs
def generate_external_vnfds(self, base_path, vnf_ids):
vnf_folders = filter(
lambda file: os.path.isdir(os.path.join(base_path, file)) and file
in vnf_ids, os.listdir(base_path))
pcs = []
for vnf in vnf_folders:
pc_entries = self.generate_vnfd_entry(os.path.join(base_path, vnf),
vnf)
if not pc_entries or len(pc_entries) == 0:
continue
for pce in iter(pc_entries):
pcs.append(pce)
return pcs
def generate_vnfd_entry(self, base_path, vnf):
"""
Compile information for a specific VNF.
The VNF descriptor is validated and added to the
package.VDU image files, referenced in the VNF
descriptor, are added to the package.
:param base_path: The path where the VNF file is located
:param vnf: The VNF reference path
:return: The package content entries.
"""
# Locate VNFD
vnfd_list = [
file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file))
and file.endswith(self._project.descriptor_extension)
]
# Validate number of Yaml files
check = len(vnfd_list)
if check == 0:
log.warning("Missing VNF descriptor file in path '{}'. "
"A descriptor with '{}' extension should be "
"in this path".format(
base_path, self._project.descriptor_extension))
return
elif check > 1:
log.warning("Multiple YAML descriptors found in '{}'. "
"Ignoring path.".format(os.path.basename(base_path)))
return
else:
with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file:
vnfd = yaml.load(_file)
vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0])
# Validate VNFD
log.debug("Validating VNF descriptor file='{}'".format(vnfd_path))
if not self._validator.validate_function(
os.path.join(base_path, vnfd_list[0])):
log.exception(
"Failed to validate VNF descriptor '{}'".format(vnfd_path))
return
# Check if this VNF exists in the ns_vnf registry.
# If does not, cancel its packaging
if not self.check_in_ns_vnf(get_vnf_id(vnfd)):
log.warning("VNF id='{}' file='{}' is not referenced in the "
"service descriptor. It will be excluded from "
"the package".format(get_vnf_id(vnfd), vnfd_path))
return
pce = []
# Create fd location
fd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(fd_path, exist_ok=True)
# Copy the descriptor file
fd = os.path.join(fd_path, vnfd_list[0])
self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd)
# Generate VNFD Entry
pce_fd = dict()
pce_fd["content-type"] = "application/sonata.function_descriptor"
pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0])
pce_fd["md5"] = generate_hash(fd)
pce.append(pce_fd)
if 'virtual_deployment_units' in vnfd:
vdu_list = [
vdu for vdu in vnfd['virtual_deployment_units']
if vdu['vm_image']
]
for vdu in vdu_list:
# vm_image can be a local File, a local Dir,
# a URL or a reference to docker image
vdu_image_path = vdu['vm_image']
if validators.url(vdu_image_path): # Check if is URL/URI.
try:
# Check if the image URL exists with a short Timeout
requests.head(vdu_image_path, timeout=1)
except (requests.Timeout, requests.ConnectionError):
log.warning("Failed to verify the "
"existence of vm_image '{}'".format(
vdu['vm_image']))
# Add image URL to artifact dependencies
self._add_artifact_dependency(
name=vnfd['name'] + '-' + vdu['id'] + '-vm_image',
vendor=vnfd['vendor'],
version=vnfd['version'],
url=vdu['vm_image'],
md5='02236f2ae558018ed14b5222ef1bd9f1')
# TODO: remote url must provide md5? This is dummy!
continue
else: # Check for URL local (e.g. file:///...)
ptokens = pathlib.Path(vdu_image_path).parts
if ptokens[0] == 'file:': # URL to local file
bd = os.path.join(base_path, ptokens[1])
else: # regular filename/path
bd = os.path.join(base_path, vdu['vm_image'])
if os.path.exists(bd): # local File or local Dir
if os.path.isfile(bd):
pce.append(
self.__pce_img_gen__(base_path,
vnf,
vdu,
vdu['vm_image'],
dir_p='',
dir_o=''))
elif os.path.isdir(bd):
for root, dirs, files in os.walk(bd):
dir_o = root[len(bd):]
dir_p = dir_o.replace(os.path.sep, "/")
for f in files:
if dir_o.startswith(os.path.sep):
dir_o = dir_o[1:]
pce.append(
self.__pce_img_gen__(root,
vnf,
vdu,
f,
dir_p=dir_p,
dir_o=dir_o))
elif vdu['vm_image_format'] == 'docker':
log.debug("Referenced vm_image is docker '{}'".format(
vdu['vm_image']))
return pce
@staticmethod
def copy_descriptor_file(src_descriptor, dst_descriptor):
"""
Copy a descriptor file. Instead of just copying the file,
it parses and reads the content of the source file, then it creates
a new file and writes in it the digested content.
:param src_descriptor:
:param dst_descriptor:
:return:
"""
with open(src_descriptor, "r") as vnfd_file:
vnf_content = yaml.load(vnfd_file)
with open(dst_descriptor, "w") as vnfd_file:
vnfd_file.write(yaml.dump(vnf_content, default_flow_style=False))
def __pce_img_gen__(self, bd, vnf, vdu, f, dir_p='', dir_o=''):
pce = dict()
img_format = 'raw' \
if not vdu['vm_image_format'] \
else vdu['vm_image_format']
pce["content-type"] = "application/sonata.{}_files".format(img_format)
pce["name"] = "/{}_files/{}{}/{}".format(img_format, vnf, dir_p, f)
pce["md5"] = self.__pce_img_gen_fc__(img_format, vnf, f, bd, dir_o)
return pce
def __pce_img_gen_fc__(self, img_format, vnf, f, root, dir_o=''):
fd_path = os.path.join("{}_files".format(img_format), vnf, dir_o)
fd_path = os.path.join(self._workdir, fd_path)
os.makedirs(fd_path, exist_ok=True)
fd = os.path.join(fd_path, f)
shutil.copyfile(os.path.join(root, f), fd)
return generate_hash(fd)
def generate_package(self, name):
"""
Generate the final package version.
:param name: The name of the final version of the package,
the project name will be used if no name provided
"""
# Validate all needed information
if not self._package_descriptor:
log.critical("Missing package descriptor. "
"Failed to generate package.")
exit(1)
if not name:
name = self._package_descriptor['vendor'] + "." + \
self._package_descriptor['name'] + "." + \
self._package_descriptor['version']
# Generate package file
zip_name = os.path.join(self._dst_path, name + '.son')
with closing(zipfile.ZipFile(zip_name, 'w')) as pck:
for base, dirs, files in os.walk(self._workdir):
for file_name in files:
full_path = os.path.join(base, file_name)
relative_path = \
full_path[len(self._workdir) + len(os.sep):]
if not full_path == zip_name:
pck.write(full_path, relative_path)
# Validate PD
log.debug("Validating Package")
if not self._validator.validate_package(zip_name):
log.debug("Failed to validate Package Descriptor. "
"Aborting package creation.")
self._package_descriptor = None
return
package_md5 = generate_hash(zip_name)
log.info("Package generated successfully.\nFile: {}\nMD5: {}\n".format(
os.path.abspath(zip_name), package_md5))
def register_ns_vnf(self, vnf_id):
"""
Add a vnf to the NS VNF registry.
:param vnf_id:
:return: True for successful registry.
False if the VNF already exists in the registry.
"""
if vnf_id in self._ns_vnf_registry:
return False
self._ns_vnf_registry[vnf_id] = False
return True
def check_in_ns_vnf(self, vnf_id):
"""Marks a VNF as packaged in the SD VNF registry."""
if vnf_id not in self._ns_vnf_registry:
return False
self._ns_vnf_registry[vnf_id] = True
return True
def get_unpackaged_ns_vnfs(self):
"""
Obtain the a list of VNFs that were referenced
by NS but weren't packaged.
"""
u_vnfs = []
for vnf in self._ns_vnf_registry:
if not self._ns_vnf_registry[vnf]:
u_vnfs.append(vnf)
return u_vnfs
def retrieve_external_vnf(self, descriptor_id):
"""
Retrieve descriptor from the service Platform catalogue.
It will loop through available Service Plaforms to retrieve the
required descriptor
:return: descriptor content
"""
# first, contact the default platform
vnfd = self._access.pull_resource('functions',
identifier=descriptor_id,
uuid=False)
if vnfd:
return vnfd
# if not retrieved, loop through remaining platforms
for platform, p_id in self._workspace.service_platforms.items():
# ignore default platform
if p_id == self._workspace.default_service_platform:
continue
vnfd = self._access.pull_resource('functions',
identifier=descriptor_id,
uuid=False,
platform_id=p_id)
if vnfd:
return vnfd
def _add_package_resolver(self,
name,
username='******',
password='******'):
log.debug("Adding package resolver entry '{}'".format(name))
# Check if already included
for pr_entry in self._package_resolvers:
if pr_entry['name'] == name:
log.debug("Package resolver entry '{}' "
"was previously added. Ignoring.".format(name))
return
pr_entry = {
'name': name,
'credentials': {
'username': username,
'password': password
}
}
self._package_resolvers.append(pr_entry)
def _add_artifact_dependency(self,
name,
vendor,
version,
url,
md5,
username='******',
password='******'):
log.debug("Adding artifact dependency entry '{}'".format(name))
# Check if already included
for ad_entry in self._artifact_dependencies:
if ad_entry['name'] == name:
log.debug("Artifact dependency entry '{}' "
"was previously added. Ignoring.".format(name))
return
ad_entry = {
'name': name,
'vendor': vendor,
'version': version,
'url': url,
'md5': md5,
'credentials': {
'username': username,
'password': password
}
}
self._artifact_dependencies.append(ad_entry)
# Set package sealed to false as it will not be self-contained
self._sealed = False
class Packager(object):
def __init__(self, workspace, project=None, services=None, functions=None,
dst_path=None, generate_pd=True, version="1.0"):
# Assign parameters
coloredlogs.install(level=workspace.log_level)
self._version = version
self._package_descriptor = None
self._workspace = workspace
self._project = project
self._services = services
self._functions = functions
# Create a son-access client
self._access = AccessClient(self._workspace,
log_level=self._workspace.log_level)
# Create a validator
self._validator = Validator(workspace=workspace)
self._validator.configure(syntax=True, integrity=False, topology=False)
# Create a schema validator
self._schema_validator = SchemaValidator(workspace)
# Keep track of VNF packaging referenced in NS
self._ns_vnf_registry = {}
# location to write the package
self._dst_path = dst_path if dst_path else '.'
# temporary working directory
self._workdir = '.package-' + str(time.time())
# Specifies THE service template of this package
self._entry_service_template = None
# Keep a list of repositories and
# catalogue servers that this package depend on.
# This will be included in the Package Resolver Section
self._package_resolvers = []
# Keep a list of external artifact
# dependencies that this package depends up on
# This will be included in the Artifact Dependencies Section
self._artifact_dependencies = []
# States if this package is self-contained,
# i.e. if contains all its relevant artifacts
self._sealed = True
# Clear and create package specific folder
if generate_pd:
self.init_package_skeleton()
self.build_package()
def init_package_skeleton(self):
"""
Validate and initialize the destination folder
for the creation of the package artifacts.
"""
if os.path.isdir(self._workdir):
log.error("Internal error. Temporary workdir already exists.")
return
# workdir
os.mkdir(self._workdir)
atexit.register(shutil.rmtree, os.path.abspath(self._workdir))
# destination path
if not os.path.isdir(self._dst_path):
os.mkdir(self._dst_path)
@property
def package_descriptor(self):
return self._package_descriptor
def build_package(self):
"""
Create and set the full package descriptor as a dictionary.
It process the file by each individual section.
"""
log.info('Create Package Content Section')
package_content = self.package_pcs()
log.info('Create Package Resolver Section')
package_resolver = self.package_prs()
log.info('Create Package Dependencies Section')
package_dependencies = self.package_pds()
log.info('Create Artifact Dependencies Section')
artifact_dependencies = self.package_ads()
# The general section must be created last,
# some fields depend on prior processing
log.info('Create General Description section')
if self._project:
general_description = self.package_gds(
prj_descriptor=self._project.project_config)
else:
general_description = self.package_gds()
if not general_description:
log.error("Failed to package General Description Section.")
return
# Compile all sections in package descriptor
self._package_descriptor = general_description
if not package_content:
log.error("Failed to package Package Content Section. "
"Could not find a network service and/or its "
"referenced function descriptors")
self._package_descriptor = None
return
self._package_descriptor.update(package_content)
self._package_descriptor.update(package_resolver)
self._package_descriptor.update(package_dependencies)
self._package_descriptor.update(artifact_dependencies)
# Create the manifest folder and file
meta_inf = os.path.join(self._workdir, "META-INF")
os.makedirs(meta_inf, exist_ok=True)
with open(os.path.join(meta_inf, "MANIFEST.MF"), "w") as manifest:
manifest.write(yaml.dump(self.package_descriptor,
default_flow_style=False))
@performance
def package_gds(self, prj_descriptor=None):
"""
Compile information for the General Description Section.
This section is exclusively filled by the project descriptor
file located on the root of every project.
"""
# List of mandatory fields to be included in the GDS
gds_fields = ['vendor', 'name', 'version',
'maintainer', 'description']
gds = dict()
gds['descriptor_version'] = self._version
gds['schema'] = self._schema_validator.get_remote_schema(
SchemaValidator.SCHEMA_PACKAGE_DESCRIPTOR)
gds['sealed'] = self._sealed
if prj_descriptor:
gds['entry_service_template'] = self._entry_service_template
if 'package' not in prj_descriptor.keys():
log.error("Please define 'package' section in {}"
.format(Project.__descriptor_name__))
return
errors = []
for field in gds_fields:
if field not in prj_descriptor['package'].keys():
errors.append(field)
else:
gds[field] = prj_descriptor['package'][field]
if errors:
log.error('Please define {} in the package section of {}'
.format(', '.join(errors),
Project.__descriptor_name__))
return
else:
# TODO: what properties to set in a custom package? TBD...
gds['vendor'] = 'custom'
gds['name'] = 'package'
gds['version'] = '1.0'
gds['maintainer'] = 'developer'
gds['description'] = 'custom generated package'
return gds
@performance
def package_pcs(self):
"""
Compile information for the Package Content Section.
This section contains all the artifacts that are
contained and shipped by the package.
"""
pcs = []
# Load and add service descriptor
if self._project:
nsd = self.generate_project_nsd()
if not nsd or len(nsd) == 0:
log.error("Failed to package service descriptor")
return
pcs += nsd
elif self._services:
nsds = self.generate_custom_nsds()
if not nsds:
log.error("Failed to package service descriptors")
return
pcs += nsds
# Load and add the function descriptors
if self._project:
vnfds = self.generate_project_vnfds()
if not vnfds or len(vnfds) == 0:
log.error("Failed to package function descriptors")
return
pcs += vnfds
elif self._functions:
vnfds = self.generate_custom_vnfds()
if not vnfds:
log.error("Failed to package function descriptors")
return
pcs += vnfds
return dict(package_content=pcs)
@performance
def package_prs(self):
"""
Compile information for the Package Resolver Section.
This section contains information about catalogues
and repositories needed to resolve the dependencies
specified in this package descriptor.
"""
if len(self._package_resolvers) == 0:
log.debug("There are no required Package Resolvers. "
"This section will not be included.")
return dict()
return dict(package_resolvers=self._package_resolvers)
@performance
def package_pds(self):
"""
Compile information for the Package Dependencies
Section. This section specifies additional packages
that this package depends up on.
"""
log.debug("There are no required Package Dependencies. "
"This section will not be included.")
return dict()
@performance
def package_ads(self):
"""
Compile information for the Artifact Dependencies
Section. This section contains components that are
not included in the package but are referenced in
its descriptors. For instance, it includes the url
of vm_images used by network functions.
"""
if len(self._artifact_dependencies) == 0:
log.debug("There are no required Artifact Dependencies. "
"This section will not be included.")
return dict()
return dict(artifact_dependencies=self._artifact_dependencies)
def generate_project_nsd(self):
"""
Compile information for the service descriptor section.
"""
base_path = os.path.join(self._project.project_root, 'sources', 'nsd')
if not os.path.isdir(base_path):
log.error("Missing NS directory '{}'".format(base_path))
return
# Ensure that only one NS descriptor exists
nsd_list = [file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file)) and
file.endswith(self._project.descriptor_extension)]
check = len(nsd_list)
if check == 0:
log.error("Missing NS Descriptor file.")
return
elif check > 1:
log.error("Only one NS Descriptor file is allowed.")
return
else:
nsd_filename = nsd_list[0]
with open(os.path.join(base_path, nsd_filename), 'r') as _file:
nsd = yaml.load(_file)
# Validate NSD
log.debug("Validating Service Descriptor NSD='{}'"
.format(nsd_filename))
if not self._validator.validate_service(os.path.join(base_path,
nsd_filename)):
log.error("Failed to validate Service Descriptor '{}'. "
"Aborting package creation".format(nsd_filename))
return
# Cycle through VNFs and register their IDs for later dependency check
if 'network_functions' in nsd:
vnf_list = \
[vnf for vnf in nsd['network_functions'] if vnf['vnf_name']]
for vnf in vnf_list:
self.register_ns_vnf(get_vnf_id_full(vnf['vnf_vendor'],
vnf['vnf_name'],
vnf['vnf_version']))
# Create SD location
nsd = os.path.join(base_path, nsd_filename)
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptor file
sd = os.path.join(sd_path, nsd_filename)
self.copy_descriptor_file(nsd, sd)
# Generate NSD package content entry
pce = []
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_filename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
# Specify the NSD as THE entry service template of package descriptor
self._entry_service_template = pce_sd['name']
return pce
def generate_custom_nsds(self):
"""
Compile information for the service descriptors, when creating a
custom package.
"""
log.info("Packaging service descriptors...")
for nsd_filename in self._services:
if not self._validator.validate_service(nsd_filename):
log.error("Failed to package service '{}'"
.format(nsd_filename))
return
# Create SD location
sd_path = os.path.join(self._workdir, "service_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy service descriptors and generate their entry points
pce = []
for nsd_filename in self._services:
nsd_basename = os.path.basename(nsd_filename)
sd = os.path.join(sd_path, nsd_basename)
self.copy_descriptor_file(nsd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.service_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(nsd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def generate_project_vnfds(self):
"""
Compile information for the function descriptors, when packaging an
SDK project.
"""
# Add VNFs from project source
log.info("Packaging VNF descriptors from project source...")
pcs = self.generate_project_source_vnfds(os.path.join(
self._project.project_root, 'sources', 'vnf'))
# Verify that all VNFs from NSD were packaged
unpack_vnfs = self.get_unpackaged_ns_vnfs()
if len(unpack_vnfs) > 0:
# Load function descriptors (VNFDs) from external sources
log.info("Solving dependencies for VNF descriptors...")
if not self.load_external_vnfds(unpack_vnfs):
log.error("Unable to solve all dependencies "
"required by the service descriptor.")
return
log.info("Packaging VNF descriptors from external source...")
pcs_ext = self.generate_external_vnfds(os.path.join(
self._workspace.workspace_root,
self._workspace.vnf_catalogue_dir),
unpack_vnfs)
if not pcs_ext or len(pcs_ext) == 0:
return
pcs += pcs_ext
# Verify again if all VNFs were correctly packaged
unpack_vnfs = self.get_unpackaged_ns_vnfs()
if len(unpack_vnfs) > 0:
log.error("Unable to validate all VNFs "
"required by the service descriptor.")
return
return pcs
def generate_custom_vnfds(self):
"""
Compile information for the function descriptors, when creating a
custom package.
"""
log.info("Packaging VNF descriptors...")
for vnfd_filename in self._functions:
if not self._validator.validate_function(vnfd_filename):
log.error("Failed to package function '{}'"
.format(vnfd_filename))
return
# Create FD location
sd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(sd_path, exist_ok=True)
# Copy function descriptors and generate their entry points
pce = []
for vnfd_filename in self._functions:
vnfd_basename = os.path.basename(vnfd_filename)
sd = os.path.join(sd_path, vnfd_basename)
self.copy_descriptor_file(vnfd_filename, sd)
pce_sd = dict()
pce_sd["content-type"] = "application/sonata.function_descriptor"
pce_sd["name"] = "/service_descriptors/{}".format(vnfd_basename)
pce_sd["md5"] = generate_hash(sd)
pce.append(pce_sd)
return pce
def load_external_vnfds(self, vnf_id_list):
"""
This method is responsible to load all VNFs, required
by the NS, that are not part of project source.
VNFs can be loaded from the Workspace catalog or/and
from the catalogue servers.
:param vnf_id_list: List of VNF ID to solve
:return: True for success, False for failure
"""
log.debug("Loading the following VNF descriptors: {}"
.format(vnf_id_list))
# Iterate through the VNFs required by the NS
for vnf_id in vnf_id_list:
log.debug("Probing workspace catalogue for VNF id='{}'..."
.format(vnf_id))
# >> First, check if this VNF is in the workspace catalogue
catalogue_path = os.path.join(
self._workspace.workspace_root,
self._workspace.vnf_catalogue_dir,
vnf_id)
if os.path.isdir(catalogue_path):
# Exists! Save catalogue path of this vnf for later packaging
log.debug("Found VNF id='{}' in workspace catalogue '{}'"
.format(vnf_id, catalogue_path))
continue
log.debug("VNF id='{}' is not present in workspace catalogue. "
"Contacting SP Catalogue...".format(vnf_id))
# If not in WS catalogue, get the VNF from the SP Catalogues
vnfd = None
self.retrieve_external_vnf(vnf_id)
if not vnfd:
log.warning("VNF id='{}' is not present in SP Catalogue"
.format(vnf_id))
return False
# Create dir to hold the retrieved VNF in workspace catalogue
log.debug("VNF id='{}' retrieved from the SP Catalogue. "
"Loading to workspace cache.".format(vnf_id))
os.mkdir(catalogue_path)
vnfd_f = open(os.path.join(catalogue_path,
vnfd['name'] +
"." +
self._project.descriptor_extension),
'w')
yaml.dump(vnfd, vnfd_f, default_flow_style=False)
return True
def generate_project_source_vnfds(self, base_path):
"""
Compile information for the list of VNFs
This function iterates over the different VNF entries
:param base_path: base dir location of VNF descriptors
:return:
"""
vnf_folders = filter(
lambda file: os.path.isdir(os.path.join(base_path, file)),
os.listdir(base_path))
pcs = []
for vnf in vnf_folders:
pc_entries = self.generate_vnfd_entry(
os.path.join(base_path, vnf), vnf)
if not pc_entries or len(pc_entries) == 0:
continue
for pce in iter(pc_entries):
pcs.append(pce)
return pcs
def generate_external_vnfds(self, base_path, vnf_ids):
vnf_folders = filter(
lambda file: os.path.isdir(os.path.join(base_path, file)) and
file in vnf_ids, os.listdir(base_path))
pcs = []
for vnf in vnf_folders:
pc_entries = self.generate_vnfd_entry(os.path.join(
base_path, vnf), vnf)
if not pc_entries or len(pc_entries) == 0:
continue
for pce in iter(pc_entries):
pcs.append(pce)
return pcs
def generate_vnfd_entry(self, base_path, vnf):
"""
Compile information for a specific VNF.
The VNF descriptor is validated and added to the
package.VDU image files, referenced in the VNF
descriptor, are added to the package.
:param base_path: The path where the VNF file is located
:param vnf: The VNF reference path
:return: The package content entries.
"""
# Locate VNFD
vnfd_list = [file for file in os.listdir(base_path)
if os.path.isfile(os.path.join(base_path, file)) and
file.endswith(self._project.descriptor_extension)]
# Validate number of Yaml files
check = len(vnfd_list)
if check == 0:
log.warning("Missing VNF descriptor file in path '{}'. "
"A descriptor with '{}' extension should be "
"in this path"
.format(base_path,
self._project.descriptor_extension))
return
elif check > 1:
log.warning("Multiple YAML descriptors found in '{}'. "
"Ignoring path.".format(os.path.basename(base_path)))
return
else:
with open(os.path.join(base_path, vnfd_list[0]), 'r') as _file:
vnfd = yaml.load(_file)
vnfd_path = os.path.join(os.path.basename(base_path), vnfd_list[0])
# Validate VNFD
log.debug("Validating VNF descriptor file='{}'".format(vnfd_path))
if not self._validator.validate_function(os.path.join(base_path,
vnfd_list[0])):
log.exception("Failed to validate VNF descriptor '{}'"
.format(vnfd_path))
return
# Check if this VNF exists in the ns_vnf registry.
# If does not, cancel its packaging
if not self.check_in_ns_vnf(get_vnf_id(vnfd)):
log.warning("VNF id='{}' file='{}' is not referenced in the "
"service descriptor. It will be excluded from "
"the package"
.format(get_vnf_id(vnfd), vnfd_path))
return
pce = []
# Create fd location
fd_path = os.path.join(self._workdir, "function_descriptors")
os.makedirs(fd_path, exist_ok=True)
# Copy the descriptor file
fd = os.path.join(fd_path, vnfd_list[0])
self.copy_descriptor_file(os.path.join(base_path, vnfd_list[0]), fd)
# Generate VNFD Entry
pce_fd = dict()
pce_fd["content-type"] = "application/sonata.function_descriptor"
pce_fd["name"] = "/function_descriptors/{}".format(vnfd_list[0])
pce_fd["md5"] = generate_hash(fd)
pce.append(pce_fd)
if 'virtual_deployment_units' in vnfd:
vdu_list = [vdu for vdu in vnfd['virtual_deployment_units']
if vdu['vm_image']]
for vdu in vdu_list:
# vm_image can be a local File, a local Dir,
# a URL or a reference to docker image
vdu_image_path = vdu['vm_image']
if validators.url(vdu_image_path): # Check if is URL/URI.
try:
# Check if the image URL exists with a short Timeout
requests.head(vdu_image_path, timeout=1)
except (requests.Timeout, requests.ConnectionError):
log.warning("Failed to verify the "
"existence of vm_image '{}'"
.format(vdu['vm_image']))
# Add image URL to artifact dependencies
self._add_artifact_dependency(
name=vnfd['name'] + '-' + vdu['id'] + '-vm_image',
vendor=vnfd['vendor'],
version=vnfd['version'],
url=vdu['vm_image'],
md5='02236f2ae558018ed14b5222ef1bd9f1')
# TODO: remote url must provide md5? This is dummy!
continue
else: # Check for URL local (e.g. file:///...)
ptokens = pathlib.Path(vdu_image_path).parts
if ptokens[0] == 'file:': # URL to local file
bd = os.path.join(base_path, ptokens[1])
else: # regular filename/path
bd = os.path.join(base_path, vdu['vm_image'])
if os.path.exists(bd): # local File or local Dir
if os.path.isfile(bd):
pce.append(self.__pce_img_gen__(
base_path, vnf, vdu, vdu['vm_image'],
dir_p='', dir_o=''))
elif os.path.isdir(bd):
for root, dirs, files in os.walk(bd):
dir_o = root[len(bd):]
dir_p = dir_o.replace(os.path.sep, "/")
for f in files:
if dir_o.startswith(os.path.sep):
dir_o = dir_o[1:]
pce.append(self.__pce_img_gen__(
root, vnf, vdu, f,
dir_p=dir_p, dir_o=dir_o))
elif vdu['vm_image_format'] == 'docker':
log.debug("Referenced vm_image is docker '{}'"
.format(vdu['vm_image']))
return pce
@staticmethod
def copy_descriptor_file(src_descriptor, dst_descriptor):
"""
Copy a descriptor file. Instead of just copying the file,
it parses and reads the content of the source file, then it creates
a new file and writes in it the digested content.
:param src_descriptor:
:param dst_descriptor:
:return:
"""
with open(src_descriptor, "r") as vnfd_file:
vnf_content = yaml.load(vnfd_file)
with open(dst_descriptor, "w") as vnfd_file:
vnfd_file.write(yaml.dump(vnf_content, default_flow_style=False))
def __pce_img_gen__(self, bd, vnf, vdu, f, dir_p='', dir_o=''):
pce = dict()
img_format = 'raw' \
if not vdu['vm_image_format'] \
else vdu['vm_image_format']
pce["content-type"] = "application/sonata.{}_files".format(img_format)
pce["name"] = "/{}_files/{}{}/{}".format(img_format, vnf, dir_p, f)
pce["md5"] = self.__pce_img_gen_fc__(img_format, vnf, f, bd, dir_o)
return pce
def __pce_img_gen_fc__(self, img_format, vnf, f, root, dir_o=''):
fd_path = os.path.join("{}_files".format(img_format), vnf, dir_o)
fd_path = os.path.join(self._workdir, fd_path)
os.makedirs(fd_path, exist_ok=True)
fd = os.path.join(fd_path, f)
shutil.copyfile(os.path.join(root, f), fd)
return generate_hash(fd)
def generate_package(self, name):
"""
Generate the final package version.
:param name: The name of the final version of the package,
the project name will be used if no name provided
"""
# Validate all needed information
if not self._package_descriptor:
log.critical("Missing package descriptor. "
"Failed to generate package.")
exit(1)
if not name:
name = self._package_descriptor['vendor'] + "." + \
self._package_descriptor['name'] + "." + \
self._package_descriptor['version']
# Generate package file
zip_name = os.path.join(self._dst_path, name + '.son')
with closing(zipfile.ZipFile(zip_name, 'w')) as pck:
for base, dirs, files in os.walk(self._workdir):
for file_name in files:
full_path = os.path.join(base, file_name)
relative_path = \
full_path[len(self._workdir) + len(os.sep):]
if not full_path == zip_name:
pck.write(full_path, relative_path)
# Validate PD
log.debug("Validating Package")
if not self._validator.validate_package(zip_name):
log.debug("Failed to validate Package Descriptor. "
"Aborting package creation.")
self._package_descriptor = None
return
package_md5 = generate_hash(zip_name)
log.info("Package generated successfully.\nFile: {}\nMD5: {}\n"
.format(os.path.abspath(zip_name), package_md5))
def register_ns_vnf(self, vnf_id):
"""
Add a vnf to the NS VNF registry.
:param vnf_id:
:return: True for successful registry.
False if the VNF already exists in the registry.
"""
if vnf_id in self._ns_vnf_registry:
return False
self._ns_vnf_registry[vnf_id] = False
return True
def check_in_ns_vnf(self, vnf_id):
"""Marks a VNF as packaged in the SD VNF registry."""
if vnf_id not in self._ns_vnf_registry:
return False
self._ns_vnf_registry[vnf_id] = True
return True
def get_unpackaged_ns_vnfs(self):
"""
Obtain the a list of VNFs that were referenced
by NS but weren't packaged.
"""
u_vnfs = []
for vnf in self._ns_vnf_registry:
if not self._ns_vnf_registry[vnf]:
u_vnfs.append(vnf)
return u_vnfs
def retrieve_external_vnf(self, descriptor_id):
"""
Retrieve descriptor from the service Platform catalogue.
It will loop through available Service Plaforms to retrieve the
required descriptor
:return: descriptor content
"""
# first, contact the default platform
vnfd = self._access.pull_resource('functions',
identifier=descriptor_id,
uuid=False)
if vnfd:
return vnfd
# if not retrieved, loop through remaining platforms
for platform, p_id in self._workspace.service_platforms.items():
# ignore default platform
if p_id == self._workspace.default_service_platform:
continue
vnfd = self._access.pull_resource('functions',
identifier=descriptor_id,
uuid=False,
platform_id=p_id)
if vnfd:
return vnfd
def _add_package_resolver(self, name, username='******',
password='******'):
log.debug("Adding package resolver entry '{}'".format(name))
# Check if already included
for pr_entry in self._package_resolvers:
if pr_entry['name'] == name:
log.debug("Package resolver entry '{}' "
"was previously added. Ignoring."
.format(name))
return
pr_entry = {'name': name,
'credentials': {
'username': username,
'password': password
}}
self._package_resolvers.append(pr_entry)
def _add_artifact_dependency(self, name, vendor, version, url, md5,
username='******', password='******'):
log.debug("Adding artifact dependency entry '{}'".format(name))
# Check if already included
for ad_entry in self._artifact_dependencies:
if ad_entry['name'] == name:
log.debug("Artifact dependency entry '{}' "
"was previously added. Ignoring."
.format(name))
return
ad_entry = {'name': name,
'vendor': vendor,
'version': version,
'url': url,
'md5': md5,
'credentials': {
'username': username,
'password': password
}}
self._artifact_dependencies.append(ad_entry)
# Set package sealed to false as it will not be self-contained
self._sealed = False
