def test_pack_packages(self):
# prepare test
args = parse_args(["--format", "eu.lf.onap"])
args.package = ""
p = PM.new_packager(args, pkg_format=args.pkg_format)
project_name = "project"
wd = tempfile.mkdtemp()
self.reset_tmp_files()
package_set = self.create_test_OnapPackageSet(project_name)
p.attach_files(package_set)
p.pack_packages(wd, package_set)
for package in package_set.packages():
package_path = os.path.join(wd,
"{}.csar".format(package.package_name))
self.assertTrue(os.path.exists(package_path),
msg=str((package_path, os.listdir(wd))))
for vnf in package_set.vnfds.values():
package_path = os.path.join(wd, "{}.csar".format(vnf.package_name))
with zipfile.ZipFile(package_path) as f:
member_names = f.namelist()
for folder in OnapPackageSet.folders:
self.assertTrue(
self.substring_in_list(folder, member_names))
file_members = list(
map(lambda member: os.path.basename(member), member_names))
for file in vnf.package_content:
filename = os.path.basename(file["filename"])
self.assertIn(filename, file_members)
def test_do_unpackage_good_package(self):
self.default_args = parse_args([])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example.tgo")
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = self.p._do_unpackage()
self.assertIsNone(r.error)
def test_do_package_good_project(self):
self.default_args = parse_args([])
self.default_args.package = misc_file("5gtango_ns_project_example1")
self.default_args.output = os.path.join(tempfile.mkdtemp(), "test.tgo")
p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = p._do_package()
self.assertIsNone(r.error)
# check structure of wd
wd = r._project_wd
self.assertTrue(os.path.exists(wd))
self.assertTrue(os.path.exists(os.path.join(wd, "TOSCA-Metadata")))
self.assertTrue(os.path.exists(os.path.join(wd, "Definitions")))
for pc in r.package_content:
self.assertTrue(
os.path.exists(
os.path.join(wd, os.path.dirname(pc.get("source")))))
# check generated files
for pc in r.package_content:
self.assertTrue(os.path.exists(os.path.join(wd, pc.get("source"))))
self.assertTrue(
os.path.exists(os.path.join(wd, "TOSCA-Metadata/NAPD.yaml")))
self.assertTrue(os.path.exists(os.path.join(wd, "etsi_manifest.mf")))
self.assertTrue(
os.path.exists(os.path.join(wd, "TOSCA-Metadata/TOSCA.meta")))
# check *.tgo file
self.assertTrue(os.path.exists(self.default_args.output))
def run(args=None):
"""
Entry point.
Can get a list of args that are then
used as input for the CLI arg parser.
"""
if sys.version_info[0] < 3:
print("tng-pkg can only run on Python3!")
print("But you are using Python{}".format(sys.version_info[0]))
print("Abort.")
sys.exit(1)
args = cli.parse_args(args)
setup_logging(args)
if args.dump_swagger:
rest.dump_swagger(args)
LOG.info("Dumped Swagger API model to {}".format(
args.dump_swagger_path))
exit(0)
# TODO validate if args combination makes any sense
if args.service:
# start tng-sdk-package in service mode (REST API)
rest.serve_forever(args)
else:
# run package in CLI mode
return cli.dispatch(args)
return None
def test_do_unpackage_bad_checksum(self):
self.default_args = parse_args([])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example-bad-checksum.tgo")
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = self.p._do_unpackage()
self.assertIsNotNone(r.error)
self.assertIn("Checksum mismatch!", r.error)
def test_do_unpackage_bad_metadata(self):
self.default_args = parse_args([])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example-malformed.tgo")
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = self.p._do_unpackage()
self.assertIsNotNone(r.error)
self.assertIn("failed", r.error)
def test_direct_call_package_direct_call_unpackage(self):
# 1. package
self.default_args = parse_args([])
self.default_args.package = misc_file("5gtango_ns_project_example1")
self.default_args.output = os.path.join(tempfile.mkdtemp(), "test.tgo")
pkg_path = self.default_args.output
p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = p._do_package()
self.assertIsNone(r.error)
# check *.tgo file
self.assertTrue(os.path.exists(self.default_args.output))
# 2. unpackage
self.default_args = parse_args([])
self.default_args.unpackage = pkg_path
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = self.p._do_unpackage()
self.assertIsNone(r.error)
def test_do_unpackage_bad_package(self):
self.default_args = parse_args([])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example-bad.tgo")
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = self.p._do_unpackage()
self.assertIsNotNone(r.error)
self.assertIn("tng-validate error", r.error)
def setUp(self):
# we need a packager to setup a environment to work on
self.default_args = parse_args(["-o", tempfile.mkdtemp()])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example.tgo")
self.p = PM.new_packager(self.default_args,
pkg_format="eu.5gtango",
storage_backend=None)
def setUp(self):
# configure mocks
self.patcher = patch("requests.post", mock_requests_post)
self.patcher.start()
# configure flask
app.config['TESTING'] = True
app.cliargs = parse_args([])
self.app = app.test_client()
def test_cli_package_auto_name(self):
# specify output dir. but not file name
pkg_path = tempfile.mkdtemp()
args = cli.parse_args(
["-p", "misc/5gtango_ns_project_example1/", "-o", pkg_path])
r = cli.dispatch(args)
self.assertIsNone(r.error)
self.assertTrue(os.path.exists(pkg_path))
shutil.rmtree(pkg_path)
def test_cli_unpackage_invalid(self):
tempdir = tempfile.mkdtemp()
args = cli.parse_args([
"-u", "misc/5gtango-ns-package-example-malformed.tgo", "-o",
tempdir
])
r = cli.dispatch(args)
self.assertIsNotNone(r.error)
shutil.rmtree(tempdir)
def test_package_unpackage_with_subfolder_compression(self):
self.default_args = parse_args([])
self.default_args.package = misc_file(
"mixed-ns-project-subfolder-test")
self.default_args.output = os.path.join(tempfile.mkdtemp(), "test.tgo")
pkg_path = self.default_args.output
p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = p._do_package()
self.assertIsNone(r.error)
# check *.tgo file
self.assertTrue(os.path.exists(self.default_args.output))
pp = misc_file("mixed-ns-project-subfolder-test")
subfolder_files = os.path.join(pp, "subfolder")
subfolder_files = get_files(subfolder_files)
tmp = tempfile.mkdtemp()
with zipfile.ZipFile(pkg_path) as zf:
zf.extract("subfolder.zip", path=tmp)
with zipfile.ZipFile(os.path.join(tmp, "subfolder.zip")) as zf:
names = zf.namelist()
names = [os.path.basename(file) for file in names]
for file in subfolder_files:
self.assertIn(file, names)
self.default_args = parse_args([])
self.default_args.unpackage = pkg_path
self.default_args.output = tempfile.mkdtemp()
self.p = PM.new_packager(self.default_args,
pkg_format="eu.5gtango",
storage_backend=TangoProjectFilesystemBackend(
self.default_args))
r = self.p._do_unpackage()
storage_location = r.metadata.get("_storage_location")
self.assertIsNone(r.error)
self.assertTrue(os.path.exists(storage_location), msg=storage_location)
self.assertTrue(os.path.exists(
os.path.join(storage_location, "subfolder")),
msg=os.listdir(storage_location))
self.assertTrue(
os.path.isdir(os.path.join(storage_location, "subfolder")))
names = get_files(os.path.join(storage_location, "subfolder"))
for file in subfolder_files:
self.assertIn(file, names)
def test_cli_package_fixed_name(self):
tempdir = tempfile.mkdtemp()
# specify a fixed name for the output
pkg_path = os.path.join(tempdir, "package.tgo")
args = cli.parse_args(
["-p", "misc/5gtango_ns_project_example1/", "-o", pkg_path])
r = cli.dispatch(args)
self.assertIsNone(r.error)
self.assertTrue(os.path.exists(pkg_path))
shutil.rmtree(tempdir)
def test_do_package_good_project_with_autoversion(self):
# set up test
self.default_args = parse_args([])
tmpdir = tempfile.mkdtemp()
tmp_project = os.path.join(tmpdir, "5gtango_ns_project_example1")
tmp_project = copytree(misc_file("5gtango_ns_project_example1"),
tmp_project)
self.default_args.package = tmp_project
self.default_args.output = os.path.join(tmpdir, "test.tgo")
self.default_args.autoversion = True
with open(os.path.join(self.default_args.package, "project.yml"),
"r") as f:
old_diction = yaml.load(f)
old_version = LooseVersionExtended(old_diction["package"]["version"])
p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = p._do_package()
autoversioned = p.autoversion(old_diction)
self.assertIsNone(r.error)
# check structure of wd
wd = r._project_wd
self.assertTrue(os.path.exists(wd))
self.assertTrue(os.path.exists(os.path.join(wd, "TOSCA-Metadata")))
# check NAPD.yaml
self.assertTrue(
os.path.exists(os.path.join(wd, "TOSCA-Metadata/NAPD.yaml")))
with open(os.path.join(wd, "TOSCA-Metadata/NAPD.yaml")) as f:
new_diction = yaml.load(f)
self.assertTrue(
LooseVersionExtended(new_diction["version"]) > old_version)
self.assertEqual(new_diction["version"],
autoversioned["package"]["version"])
# check etsi
self.assertTrue(os.path.exists(os.path.join(wd, "etsi_manifest.mf")))
with open(os.path.join(wd, "etsi_manifest.mf"), "r") as f:
new_diction = yaml.load(f)
self.assertTrue(
LooseVersionExtended(new_diction["ns_package_version"]) >
old_version)
self.assertEqual(new_diction["ns_package_version"],
autoversioned["package"]["version"])
# check *.tgo file
self.assertTrue(os.path.exists(self.default_args.output))
# check source file
with open(os.path.join(self.default_args.package, "project.yml"),
"r") as f:
new_project_descriptor = yaml.load(f)
self.assertEqual(new_project_descriptor, autoversioned)
self.assertTrue(
LooseVersionExtended(new_project_descriptor["package"]["version"])
> old_version)
def setUp(self):
# configure mocks
self.patcher = patch("requests.post", mock_requests_post)
# we need a packager to setup a environment to work on
self.default_args = parse_args([])
self.default_args.unpackage = misc_file(
"5gtango-ns-package-example.tgo")
self.p = PM.new_packager(self.default_args,
pkg_format="eu.5gtango",
storage_backend=None)
# patch the requests lib to not do real requests
self.patcher.start()
def test_do_package_bad_project(self):
self.default_args = parse_args([])
self.default_args.package = misc_file(
"5gtango_ns_project_example1_bad")
self.default_args.output = os.path.join(tempfile.mkdtemp(), "test.tgo")
p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
r = p._do_package()
self.assertIsNotNone(r.error)
self.assertIn("tng-validate error", r.error)
# self.assertIn(
# "Failed to read service function descriptors", r.error)
# check *.tgo file
self.assertFalse(os.path.exists(self.default_args.output))
def test_cli_unpackage(self):
tempdir = tempfile.mkdtemp()
args = cli.parse_args([
"-u", "misc/5gtango-ns-package-example.tgo", "-o", tempdir,
"--store-backend", "TangoProjectFilesystemBackend"
])
r = cli.dispatch(args)
self.assertIsNone(r.error)
self.assertTrue(
os.path.exists(
os.path.join(tempdir,
"5gtango-ns-package-example/project.yml")))
shutil.rmtree(tempdir)
def test_do_package(self):
# prepare test
project = misc_file("mixed-ns-project")
output = tempfile.mkdtemp()
args = parse_args(
["--format", "eu.lf.onap", "-p", project, "-o", output])
p = PM.new_packager(args, pkg_format=args.pkg_format)
# execute
p._do_package()
packages = os.listdir(output)
self.assertEqual(len(packages), 2)
for package in packages:
self.assertEqual(os.path.splitext(package)[1], ".csar")
self.assertTrue(self.substring_in_list("onap_nsd", packages),
msg="onap_nsd not as substr in {}".format(packages))
self.assertTrue(self.substring_in_list("onap_vnfd", packages),
msg="onap_vnfd not as substr in {}".format(packages))
with open(os.path.join(project, "project.yml")) as f:
pd = yaml.load(f)
files = pd["files"]
files = [
os.path.basename(file["path"]) for file in files
if "onap" in file["type"] or "lf.onap" in file["tags"]
]
nsd = None
vnfd = None
for file in files:
if "nsd" in file:
nsd = file
if "vnfd" in file:
vnfd = file
files.remove(nsd)
files.remove(vnfd)
for package in packages:
with zipfile.ZipFile(os.path.join(output, package)) as zf:
names = zf.namelist()
for file in files:
self.assertTrue(self.substring_in_list(file, names),
msg="{} not in {}".format(file, names))
if "nsd" in package:
self.assertIn(nsd, names)
self.assertIn(os.path.splitext(nsd)[0] + ".mf", names)
if "vnfd" in package:
self.assertIn(vnfd, names)
self.assertIn(os.path.splitext(vnfd)[0] + ".mf", names)
self.assertIn(os.path.join("TOSCA-Metadata", "TOSCA.meta"),
names)
def test_store_idempotent(self):
self.default_args = parse_args(["-o", tempfile.mkdtemp()])
self.default_args.unpackage = misc_file(
"eu.5gtango.idempotency_test.0.1.tgo")
self.p = PM.new_packager(self.default_args,
pkg_format="eu.5gtango",
storage_backend=None)
tpb = TangoProjectFilesystemBackend(self.default_args)
napdr = self.p._do_unpackage()
wd = napdr.metadata.get("_napd_path").replace(
"/TOSCA-Metadata/NAPD.yaml", "")
# call store using active working dir
new_napdr = tpb.store(napdr, wd, self.default_args.unpackage)
# check result
self.assertIsNotNone(new_napdr.metadata.get("_storage_location"))
sl = new_napdr.metadata.get("_storage_location")
# check created project
pd = self.default_args.output
self.assertTrue(os.path.exists(pd))
self.assertTrue(
os.path.exists(
os.path.join(pd,
"eu.5gtango.idempotency_test.0.1/project.yml")))
self.assertTrue(os.path.exists(os.path.join(sl, "project.yml")))
# files in root dir
self.assertTrue(os.path.exists(os.path.join(sl, "vnfd-a10-3.yml")))
self.assertTrue(os.path.exists(os.path.join(sl, "vnfd-nginx-3.yml")))
# files in sources/
self.assertTrue(os.path.exists(os.path.join(sl, "sources/")))
self.assertTrue(
os.path.exists(os.path.join(sl, "sources/vnfd-a10-4.yml")))
self.assertTrue(
os.path.exists(os.path.join(sl, "sources/vnfd-nginx-4.yml")))
# files in sources/[...]/
self.assertTrue(os.path.exists(os.path.join(sl,
"sources/nsd/nsd.yml")))
self.assertTrue(
os.path.exists(os.path.join(sl, "sources/vnfd/vnfd-a10.yml")))
self.assertTrue(
os.path.exists(os.path.join(sl, "sources/vnfd/vnfd-nginx.yml")))
# files in other folders of root dir
self.assertTrue(os.path.exists(os.path.join(sl, "Definitions/")))
self.assertTrue(
os.path.exists(os.path.join(sl, "Definitions/vnfd-a10-2.yml")))
self.assertTrue(
os.path.exists(os.path.join(sl, "Definitions/vnfd-nginx-2.yml")))
shutil.rmtree(pd)
def test_pack_package_source_path(self):
inputs = [{
"tags": []
}, {
"tags": ["lf.onap"]
}, {
"tags": ["lf.onap", "onap-target:new"]
}, {
"tags": ["lf.onap", "onap-target:new/bla/here"]
}]
outputs = ["", "Artifacts", "new", "new/bla/here"]
args = parse_args([])
p = OnapPackager(args)
for inp, out in zip(inputs, outputs):
self.assertEqual(p._pack_package_source_path(inp), out)
def test_attach_files(self):
args = parse_args(["--format", "eu.etsi.osm"])
args.package = ""
p = PM.new_packager(args, pkg_format=args.pkg_format)
project_name = "project"
self.reset_tmp_files()
package_set = self.create_test_OsmPackageSet(project_name)
p.attach_files(package_set)
for package in package_set.packages():
for file in package.package_content:
self.assertTrue(
os.path.exists(
os.path.join(package.temp_dir, file["source"],
file["filename"])))
def test_create_temp_dir(self):
# prepare test
args = parse_args(["--format", "eu.etsi.osm"])
p = PM.new_packager(args, pkg_format=args.pkg_format)
tmp_descriptor = tempfile.NamedTemporaryFile()
kwargs = {
"subdir_name": "name",
"descriptor": tmp_descriptor.name,
"hash": "hash_value",
"folders": OsmPackager.folders_nsd,
"project_path": "",
"checks_filename": "checksums.txt"
}
# execute
temp = p.create_temp_dir(**kwargs)
# check results
self.assertIsInstance(temp, str) # got path
self.assertTrue(os.path.exists(temp)) # created directory exists
for folder in OsmPackager.folders_nsd + [tmp_descriptor.name]:
# folders created ?
self.assertTrue(os.path.exists(os.path.join(temp, folder)),
msg=os.path.join(temp, folder))
# new descriptor file for next test
tmp_descriptor.close()
del tmp_descriptor
tmp_descriptor = tempfile.NamedTemporaryFile()
kwargs["descriptor"] = tmp_descriptor.name
kwargs["folders"] = OsmPackager.folders_vnf # vnf dir-tree this time
# execute
temp = p.create_temp_dir(**kwargs)
self.assertIsInstance(temp, str)
self.assertTrue(os.path.exists(temp))
for folder in OsmPackager.folders_vnf + [tmp_descriptor.name]:
self.assertTrue(os.path.exists(os.path.join(temp, folder)),
msg=os.path.join(temp, folder))
tmp_descriptor.close()
del tmp_descriptor
def test_do_package(self):
# prepare test
project = misc_file("mixed-ns-project")
output = tempfile.mkdtemp()
args = parse_args(
["--format", "eu.etsi.osm", "-p", project, "-o", output])
p = PM.new_packager(args, pkg_format=args.pkg_format)
# execute
napdr = p._do_package()
# checks
self.assertIsNone(napdr.error)
packages = os.listdir(output)
basename = os.path.basename(output)
basename_ns = "_".join([basename, "osm_nsd"])
basename_vnf = "_".join([basename, "osm_vnfd"])
filename_ns = basename_ns + ".tar.gz"
filename_vnf = basename_vnf + ".tar.gz"
self.assertIn(filename_ns, packages)
self.assertIn(filename_vnf, packages)
ns = [
"osm_nsd.yaml", "checksums.txt", "icons/upb_logo.png",
"vnf_config", "scripts", "ns_config", "icons"
]
vnf = [
"osm_vnfd.yaml", "checksums.txt", "scripts", "images", "icons",
"cloud_init", "charms", "icons/upb_logo.png",
"cloud_init/cloud.init", "images/mycloudimage.ref"
]
for package in packages:
with tarfile.open(os.path.join(output, package)) as f:
member_names = list(
map(lambda member: member.name, f.getmembers()))
if "ns" in package:
for member in ns:
self.assertIn(os.path.join(basename_ns, member),
member_names)
elif "vnf" in package:
for member in vnf:
self.assertIn(os.path.join(basename_vnf, member),
member_names)
def test_do_package_subfolder(self):
# prepare test
project = misc_file("mixed-ns-project-subfolder-test")
output = tempfile.mkdtemp()
args = parse_args(
["--format", "eu.etsi.osm", "-p", project, "-o", output])
p = PM.new_packager(args, pkg_format=args.pkg_format)
# execute
napdr = p._do_package()
self.assertIsNone(napdr.error)
packages = os.listdir(output)
subfolder_files = get_files(os.path.join(project, "subfolder"))
for package in packages:
with tarfile.open(os.path.join(output, package)) as f:
member_names = list(
map(lambda member: os.path.basename(member.name),
f.getmembers()))
for file in subfolder_files:
self.assertIn(file, member_names)
def run(args=None):
"""
Entry point.
Can get a list of args that are then
used as input for the CLI arg parser.
"""
args = cli.parse_args(args)
setup_logging(args)
if args.dump_swagger:
rest.dump_swagger(args)
LOG.info("Dumped Swagger API model to {}".format(
args.dump_swagger_path))
exit(0)
# TODO validate if args combination makes any sense
if args.service:
# start tng-sdk-package in service mode (REST API)
rest.serve_forever(args)
else:
# run package in CLI mode
return cli.dispatch(args)
return None
def test_create_temp_dirs(self):
# prepare test
args = parse_args(["--format", "eu.etsi.osm"])
args.package = ""
p = PM.new_packager(args, pkg_format=args.pkg_format)
project_name = "test"
tmp_descriptor = tempfile.NamedTemporaryFile()
package_set = OsmPackagesSet(NapdRecord())
# ns descriptor
package_set.nsd = OsmPackage(
{
"_project_source": tmp_descriptor.name,
"hash": "hash_value",
"filename": tmp_descriptor.name
},
project_name=project_name,
folders=OsmPackagesSet.folders_nsd)
# 5 vnf descriptors
vnf_num = 5
tmps = [tempfile.NamedTemporaryFile() for i in range(vnf_num)]
package_set.vnfds = {
tmp_descriptor.name: OsmPackage(
{
"_project_source": tmp_descriptor.name,
"hash": "hash_value" + str(i),
"filename": tmp_descriptor.name
},
project_name=project_name,
folders=OsmPackagesSet.folders_vnf)
for i, tmp_descriptor in zip(range(vnf_num), tmps)
}
# execute
p.create_temp_dirs(package_set, project_path="")
# check if directories created
for package in package_set.packages():
self.assertTrue(os.path.exists(package.temp_dir), msg=str(package))
def main():
logging_setup()
args = cli.parse_args()
# TODO better log configuration (e.g. file-based logging)
if args.verbose:
coloredlogs.install(level="DEBUG")
else:
coloredlogs.install(level="INFO")
if args.dump_swagger:
rest.dump_swagger(args)
LOG.info("Dumped Swagger API model to {}".format(
args.dump_swagger_path))
sys.exit(0)
# TODO validate if args combination makes any sense
if args.service:
# start tng-sdk-package in service mode (REST API)
rest.serve_forever(args)
else:
# run package in CLI mode
r = cli.dispatch(args)
if r.error is not None:
exit(1) # exit with error code
exit(0)
def test_pack_packages(self):
# prepare test
args = parse_args(["--format", "eu.etsi.osm"])
args.package = ""
p = PM.new_packager(args, pkg_format=args.pkg_format)
project_name = "project"
wd = tempfile.mkdtemp()
self.reset_tmp_files()
package_set = self.create_test_OsmPackageSet(project_name)
p.attach_files(package_set)
p.pack_packages(wd, package_set)
for package in package_set.packages():
package_path = os.path.join(
wd, "{}.tar.gz".format(package.package_name))
self.assertTrue(os.path.exists(package_path),
msg=str((package_path, os.listdir(wd))))
for vnf in package_set.vnfds.values():
package_path = os.path.join(wd,
"{}.tar.gz".format(vnf.package_name))
with tarfile.open(package_path) as f:
member_names = list(
map(lambda member: member.name, f.getmembers()))
for i, folder in enumerate(OsmPackagesSet.folders_vnf):
member = os.path.join(vnf._subdir, folder)
self.assertIn(member, member_names)
file_members = list(
map(lambda member: os.path.basename(member), member_names))
for file in vnf.package_content:
filename = os.path.basename(file["filename"])
self.assertIn(filename, file_members)
def setUp(self):
# list can manually define CLI arguments
self.default_args = parse_args(["--skip-validation"])
self.p = PM.new_packager(self.default_args, pkg_format="eu.5gtango")
self.assertIn("TangoPackager", str(type(self.p)))
