def test_get_project_package(self):
"""
A handful of projects were created by ocpidev in setUp and listed in
PROJECT_PACKAGES. Each one has different project name and package settings.
Verify that each is correct from different CWDs.
"""
logging.info("===========================\nTesting 'get_project_package'")
for proj, pkg in list(PROJECT_PACKAGES.items()):
logging.info("Project \"" + proj + "\" should have full-package: " + pkg)
self.assertEqual(ocpiutil.get_project_package(proj), pkg)
logging.info("---------------------------")
logging.info("Rerunning full-package name tests from " +
"within subdirs in each project.")
for proj, pkg in list(PROJECT_PACKAGES.items()):
with ocpiutil.cd(proj):
logging.info("Project \"" + proj + "\" should have full-package: " + pkg)
self.assertEqual(ocpiutil.get_project_package(), pkg)
self.assertEqual(ocpiutil.get_project_package("."), pkg)
components_dir = "components"
if not os.path.exists("components"):
components_dir = "lib/components"
if os.path.exists(components_dir):
with ocpiutil.cd(components_dir):
self.assertEqual(ocpiutil.get_project_package(), pkg)
self.assertEqual(ocpiutil.get_project_package("."), pkg)
self.assertEqual(ocpiutil.get_project_package(".."), pkg)
logging.info("---------------------------")
logging.info("Project package for a directory outside a project " +
"or invalid should be None.")
self.assertEqual(ocpiutil.get_project_package("/"), None)
self.assertEqual(ocpiutil.get_project_package(None), None)
def test_pj_with_pkg_exist(self):
"""
Verify that each known package is recognized as existing from different CWDs.
Invalid ones of course should result in False for 'project with pkg DNE'
"""
logging.info("===========================\n" +
"Testing 'does_project_with_package_exist'")
# TODO: test case where registry does not exist, should return false
for proj, pkg in list(PROJECT_PACKAGES.items()):
logging.info("A project (\"" + proj + "\") with package \"" + pkg +
"\" should exist.")
self.assertTrue(
ocpiutil.does_project_with_package_exist(package=pkg))
with ocpiutil.cd(proj):
logging.info("Trying from within project '" + proj + "'.")
# Exclude deep-copied/exported projects because they cannot import themselves
# or the risk infinite recursive imports
if not re.search(r".*exported$", proj):
self.assertTrue(
ocpiutil.does_project_with_package_exist(package=pkg))
logging.info("Should not have to specify package name " +
"from within project name.")
self.assertTrue(ocpiutil.does_project_with_package_exist())
self.assertFalse(
ocpiutil.does_project_with_package_exist(
package="INVALID"))
components_dir = "components"
if not os.path.exists("components"):
components_dir = "lib/components"
if os.path.exists(components_dir):
with ocpiutil.cd(components_dir):
logging.info(
"Trying from within project components library."
)
self.assertTrue(
ocpiutil.does_project_with_package_exist(
package=pkg))
self.assertTrue(
ocpiutil.does_project_with_package_exist())
self.assertTrue(
ocpiutil.does_project_with_package_exist(".."))
self.assertFalse(
ocpiutil.does_project_with_package_exist(
package="INVALID"))
logging.info("---------------------------")
logging.info("If project package is invalid," + \
"does_project_with_package_exist should return False.\n" +
"If project package is blank" +
"and the specified or current directory is not within a\n" +
"project, does_project_with_package_exist should return False.")
self.assertFalse(
ocpiutil.does_project_with_package_exist(package="INVALID"))
self.assertFalse(ocpiutil.does_project_with_package_exist("/"))
self.assertFalse(
ocpiutil.does_project_with_package_exist(package="INVALID"))
def main():
"""
Function that is called if this module is called as a mian function
"""
args = parse_cl_vars()
directory = None
name = None
try:
cur_dir = args['cur_dir']
with ocpiutil.cd(cur_dir):
dir_type = ocpiutil.get_dirtype()
# args['name'] could be None if no name is provided at the command line
name = args['name']
directory = ocpiutil.get_ocpidev_working_dir(
origin_path=".",
noun=args.get("noun", ""),
name=name,
library=args['library'],
hdl_library=args['hdl_library'],
hdl_platform=args['hdl_plat_dir'])
if (name is None) and (dir_type
in [n for n in NOUNS if n != "tests"]):
name = os.path.basename(os.path.realpath('.'))
del args['name']
if args['noun'] is None:
if dir_type in [n for n in NOUNS if n != "tests"]:
args['noun'] = dir_type
else:
raise ocpiutil.OCPIException(
"Invalid directory type \"" + str(dir_type) +
"\" Valid directory types are: " +
", ".join([n for n in NOUNS if n != "tests"]))
if ocpiutil.get_dirtype(
directory) == "libraries" and args['noun'] == "library":
args['noun'] = "libraries"
set_init_values(args, dir_type)
ocpiutil.logging.debug("creating asset as the following \nnoun: " +
args['noun'] + "\nname: " + str(name) +
"\n" + "directory: " + str(directory) +
"\nargs: " + str(args))
my_asset = ocpifactory.AssetFactory.factory(
args['noun'], directory, name, **args)
sys.exit(my_asset.run())
except ocpiutil.OCPIException as ex:
ocpiutil.logging.error(ex)
if args['noun'] is not None:
my_noun = '' if args[
'noun'] is None else " \"" + args['noun'] + "\""
my_name = '' if name is None else " named \"" + name + "\""
my_dir = '' if directory is None else " in directory \"" + directory + "\""
run_dbg = "Unable to run" + my_noun + my_name + my_dir + " due to previous errors"
else:
run_dbg = "Unable to run due to previous errors"
ocpiutil.logging.error(run_dbg)
sys.exit(1)
def setUpClass(cls):
"""
Need to make sure registry is set to current system default before proceeding
"""
if subprocess.call("mkdir -p " + UTIL_PROJ, shell=True) != 0:
raise ocpiutil.OCPIException("mkdir failed to create the utilization test project directory")
with ocpiutil.cd(UTIL_PROJ):
if subprocess.call("tar xf ../" + UTIL_PROJ + ".tgz", shell=True) != 0:
raise ocpiutil.OCPIException("could not unpack tar file for utilization test project")
process = subprocess.Popen([OCPIDEV_PATH, "set", "registry"])
results = process.communicate()
if results[1] or process.returncode != 0:
raise ocpiutil.OCPIException("'ocpidev set registry' failed in utilization test\n" +
"process.returncode: " + str(process.returncode))
def construct_asset(mode):
"""
Use the given mode/asset-type to determine the directory of an asset using the global
dictionaries above. Construct and return that asset>
"""
with ocpiutil.cd(UTIL_PROJ):
kwargs = {'output_format': "table",
'hdl_plats': ["zynq_plat", "virtex6_plat", "stratix4_plat"],
'init_libs': True, 'init_workers': True,
'init_assembs': True, 'init_hdlplats': True}
logging.debug("Constructing " + mode + " with args: " + str(kwargs))
return AssetFactory.factory(asset_type=mode,
directory=DIRS[mode],
**kwargs)
def test_reg_unreg_project(self):
"""
Unregister and re-register each project in PROJECT_PACKAGES and
confirm that the project-link is successfully removed from the
registry and then recreated.
"""
logging.info("===========================\nTesting 'register_project'")
# TODO/FIXME: Register and unregister are no longer in ocpiutil and should be tested in
# a separate unit test
# TODO: test reg cases where package is 'local', and where registry does not exist
# TODO: test reg case where project with the package already exists
# TODO: test reg case where a link conflicting with the package package exists
# TODO: test unreg case where user tries to unregister a project, but the registered
# project with that package is actually a different project
reg = AssetFactory.factory("registry", Registry.get_registry_dir("."))
gprd = reg.directory
for proj_dir, pkg in list(PROJECT_PACKAGES.items()):
logging.info("Unregistering and re-registering project \"" +
proj_dir + "\" with package \"" + pkg + "\"")
reg.remove(directory=proj_dir)
self.assertFalse(os.path.lexists(gprd + "/" + pkg))
reg.add(proj_dir)
self.assertTrue(os.path.lexists(gprd + "/" + pkg))
logging.info(
"Should be able to unregister via project package instead.")
reg.remove(pkg)
self.assertFalse(os.path.lexists(gprd + "/" + pkg))
reg.add(proj_dir)
self.assertTrue(os.path.lexists(gprd + "/" + pkg))
with ocpiutil.cd(proj_dir):
# Exclude deep-copied/exported projects because they cannot import themselves
# or the risk infinite recursive imports
if not re.search(r".*exported$", proj_dir):
logging.info("Confirming this all works from within " +
"the project using path='.'")
reg.remove(directory=".")
self.assertFalse(os.path.lexists(gprd + "/" + pkg))
reg.add(".")
self.assertTrue(os.path.lexists(gprd + "/" + pkg))
# If log level is low, disable logging to prevent the expected scary ERROR
if not OCPI_LOG_LEVEL or int(OCPI_LOG_LEVEL) < 8:
ocpiutil.OCPIUTIL_LOGGER.disabled = True
logging.info(
"Should print an ERROR and return false for invalid project_paths:"
)
self.assertRaises(ocpiutil.OCPIException, reg.remove, "INVALID")
if OCPI_LOG_LEVEL and int(OCPI_LOG_LEVEL) < 8:
ocpiutil.OCPIUTIL_LOGGER.disabled = False
def check_utilization_for_mode(self, mode):
"""
For a given mode, construct the asset of interest, get its utilization,
and check that utilization report against the golden/known-good results.
"""
my_asset = construct_asset(mode)
with ocpiutil.cd(UTIL_PROJ):
report = my_asset.get_utilization()
logging.info(
"Ensuring number of data_points matches golden dataset")
self.assertEqual(len(report.data_points), len(REPORT_GOLD[mode]))
# Iterate through these utilization report data_points and compare
# the length and contents of each with its matching golden data_point
for data_point in report.data_points:
# DATA_KEYS will tell us which key is useful for uniquely identifying
# data-points in this mode
indexing_key = DATA_KEYS[mode]
index = data_point[indexing_key]
# Get the golden data-point using the "OCPI Target" and index
gold_data_point = get_gold_data_point_by_key(
mode, index, data_point["OCPI Target"])
gold_len = len(gold_data_point)
logging.info("Ensuring # of entries in data_point " +
str(index) + " matches len \"" + str(gold_len) +
"\" from the golden dataset")
self.assertEqual(len(data_point), gold_len)
# Make sure all key/value pairs match the golden/expected data-point
for key, val in data_point.items():
gold_val = gold_data_point[key]
logging.info("Ensuring that data_point entry for key=" +
key + " matches entry \"" + str(gold_val) +
"\" in golden dataset")
if isinstance(val, list):
# If the value is a list, convert to set so ordering does not affect
# comparison
self.assertEqual(set(val), set(gold_val))
else:
self.assertEqual(val, gold_val)
logging.info(
"Doing show_utilization() to confirm success, but not checking output"
)
my_asset.show_utilization()
def get_project_package(origin_path="."):
"""
Get the Package Name of the project containing 'origin_path'.
"""
path_to_project = get_path_to_project_top(origin_path)
if path_to_project is None:
logging.debug("Path \"" + str(origin_path) +
"\" is not inside a project")
return None
# From the project top, probe the Makefile for the projectpackage
# which is printed in cdk/include/project.mk in the projectpackage rule
# if ShellProjectVars is defined
with cd(path_to_project):
project_package = None
# If the project-package-id file exists, set package-id to its contents
if os.path.isfile(path_to_project + "/project-package-id"):
with open(path_to_project + "/project-package-id",
"r") as package_id_file:
project_package = package_id_file.read().strip()
logging.debug("Read Project-ID '" + project_package +
"' from file: " + path_to_project +
"/project-package-id")
# Otherwise, ask Makefile at the project top for the ProjectPackage
if project_package is None or project_package == "":
project_vars = set_vars_from_make(
"Makefile", "projectpackage ShellProjectVars=1", "verbose")
if (not project_vars is None and 'ProjectPackage' in project_vars
and len(project_vars['ProjectPackage']) > 0):
# There is only one value associated with ProjectPackage, so get element 0
project_package = project_vars['ProjectPackage'][0]
else:
logging.error("Could not determine Package-ID of project.")
return None
return project_package
def test_set_unset_registry(self):
"""
Set and unset the registry for a project in various ways. Confirm that
the resulting imports link is correct.
"""
# TODO/FIXME: set and unset registry are no longer in ocpiutil and should be tested in
# a separate unit test
logging.info("===========================\nTesting 'set/unset_project_registry'")
#TODO: test set case where imports exists and is not a link
#TODO: test set for warning when setting to a registry that does not include CDK
with ocpiutil.cd(PROJECT0):
proj = AssetFactory.factory("project", ".")
logging.info("Make sure you can set a project's registry to the default via no args.")
proj.set_registry()
self.assertEqual(os.path.realpath("../project-registry"),
os.path.realpath(os.readlink("imports")))
# Cannot set/unset the registry for a project when it is registered
self.assertRaises(ocpiutil.OCPIException, proj.set_registry, "../../../project-registry")
self.assertRaises(ocpiutil.OCPIException, proj.unset_registry)
self.assertTrue(os.path.exists("imports"))
reg = AssetFactory.factory("registry", Registry.get_registry_dir("."))
# Remove the project from the registry and proceed with unsetting
reg.remove(package_id=proj.package_id)
proj.unset_registry()
self.assertFalse(os.path.exists("imports"))
reg.add(directory=proj.directory)
logging.info("Make sure you can set a project's registry to a given directory.")
if os.path.isdir("../../../project-registry"):
# Cannot set/unset the registry for a project when it is registered
self.assertRaises(ocpiutil.OCPIException, proj.set_registry,
"../../../project-registry")
# Remove the project from the registry and proceed with (un)setting
reg.remove(package_id=proj.package_id)
proj.set_registry("../../../project-registry")
self.assertEqual(os.path.realpath("../../../project-registry"),
os.path.realpath(os.readlink("imports")))
proj.unset_registry()
self.assertFalse(os.path.exists("imports"))
reg.add(directory=proj.directory)
else:
logging.warning("Skipping this registry test because ../../../project-registry " +
"does not exist (not run from repo?).")
reg.remove(package_id=proj.package_id)
logging.info("Make sure you cannot set a project's registry to a non-dir file. " +
"Expect an ERROR:")
self.assertRaises(ocpiutil.OCPIException, proj.set_registry, "Project.mk")
self.assertFalse(os.path.exists("imports"))
# Test unset when imports is a plain file and when it does not exist
logging.info("Make sure you cannot unset/rm an 'imports' file that is not a link. " +
"Expect an ERROR:")
open("imports", 'a').close()
#self.assertFalse(ocpiutil.unset_project_registry())
self.assertRaises(ocpiutil.OCPIException, proj.unset_registry)
self.assertTrue(os.path.isfile("imports"))
os.remove("imports")
logging.info("Make sure unsetting a project's registry succeeds and does nothing " +
"when it was already unset.")
proj.unset_registry()
self.assertFalse(os.path.exists("imports"))
# Reset project registry to default
proj.set_registry()
reg.add(directory=proj.directory)
def test_get_project_registry_dir(self):
"""
Test the value extracted for project-registry dir
given different states:
With OCPI_PROJECT_REGISTRY_DIR set
With it unset, but OCPI_CDK_DIR set
With both unset
"""
logging.info("==================================" +
"===================================")
logging.info("Asserting that 'project-registry directory' " +
"defaults to correct values\n")
reg_dir = Registry.get_registry_dir()
self.assertEqual(reg_dir, os.environ.get('OCPI_PROJECT_REGISTRY_DIR'))
# Collect all of the projects except for core
# TODO: test get_all_project cases where OCPI_PROJECT_PATH exists,
# and also where registry does not
all_pjs = [pj for pj in ocpiutil.get_all_projects() if not re.search(r".*/ocpi\.core$", pj)]
pj_paths = [os.path.realpath('.') + '/project-registry/'
+ pj for pj in list(PROJECT_PACKAGES.values())]
golden_all_pjs = pj_paths
project_path = os.environ.get('OCPI_PROJECT_PATH')
if project_path:
golden_all_pjs += project_path.split(':')
logging.info("Verifying that get_all_projects correctly collects CDK, Project path, " +
"and registry contents: " + str(golden_all_pjs))
self.assertEqual(set(all_pjs), set(golden_all_pjs))
orig_gprd = os.environ['OCPI_PROJECT_REGISTRY_DIR']
logging.info("Ensuring that invalid registry environment variable results in error")
os.environ['OCPI_PROJECT_REGISTRY_DIR'] = "INVALID"
self.assertRaises(ocpiutil.OCPIException, Registry.get_registry_dir)
logging.info("Verify that a project's imports takes precedence over the environment's " +
"registry settings.")
for proj in PROJECT_PACKAGES:
with ocpiutil.cd(proj):
reg_dir = Registry.get_registry_dir()
if re.search(r".*exported$", proj):
logging.debug("Verify that deep-copied/exported project (" + proj + ") uses " +
"its imports dir as the registry.")
self.assertEqual(os.path.realpath("imports"), reg_dir)
else:
logging.debug("Verify that the project (" + proj + ") uses its " +
"imports link as the registry.")
self.assertEqual(os.path.realpath("../project-registry"), reg_dir)
# Test for 'bad' registry results
os.environ['OCPI_PROJECT_REGISTRY_DIR'] = __file__
logging.info("Verify that the default registry is correct even for a bad registry.")
self.assertEqual(__file__, Registry.get_default_registry_dir())
self.assertRaises(ocpiutil.OCPIException, Registry.get_registry_dir)
del os.environ['OCPI_PROJECT_REGISTRY_DIR']
logging.info("Verify that the registry is correct when OCPI_PROJECT_REGISTRY_DIR is unset.")
reg_dir = Registry.get_registry_dir()
self.assertEqual(reg_dir, os.path.realpath(os.environ.get('OCPI_CDK_DIR') + "/../project-registry"))
logging.info("Verify that the default registry is correct when the env var is unset.")
self.assertEqual(reg_dir, Registry.get_default_registry_dir())
if os.path.isdir("/opt/opencpi/project-registry"):
orig_cdk = os.environ['OCPI_CDK_DIR']
del os.environ['OCPI_CDK_DIR']
reg_dir = Registry.get_registry_dir()
self.assertEqual(reg_dir, "/opt/opencpi/project-registry")
logging.info("Verify that the default registry is correct when no env vars are set.")
self.assertEqual(reg_dir, Registry.get_default_registry_dir())
os.environ['OCPI_CDK_DIR'] = orig_cdk
else:
logging.warning("Skipping default registry check because " +
"/opt/opencpi/project-registry does not exist")
os.environ['OCPI_PROJECT_REGISTRY_DIR'] = orig_gprd
def main():
"""
Function that is called if this module is called as a mian function
"""
(args, noun) = parse_cl_vars()
action = "show"
try:
cur_dir = args['cur_dir']
with ocpiutil.cd(cur_dir):
dir_type = ocpiutil.get_dirtype()
# args['name'] could be None if no name is provided at the command line
name = args.get("name", None)
if not name:
name = ""
# Now that we have grabbed name, delete it from the args that will be passed into the
# AssetFactory because name is explicitly passed to AssetFactory as a separate argument
del args['name']
check_scope_options(args.get("scope", None), noun)
if noun in ["registry", "projects"]:
directory = ocpiregistry.Registry.get_registry_dir()
elif noun not in [
"libraries", "hdlplatforms", "hdltargets", "rccplatforms",
"rcctargets", "platforms", "targets", "workers",
"components"
]:
directory = ocpiutil.get_ocpidev_working_dir(
origin_path=cur_dir,
noun=noun,
name=name,
library=args['library'],
hdl_library=args['hdl_library'],
hdl_platform=args['hdl_plat_dir'])
elif noun in ["libraries", "tests", "workers", "components"]:
if args.get("scope", None) == "local":
directory = ocpiutil.get_path_to_project_top()
elif args.get("scope", None) == "global":
directory = ocpiregistry.Registry.get_registry_dir()
else:
directory = ""
ocpiutil.logging.debug('Choose directory "' + directory +
'" to operate in')
if noun is None:
noun = dir_type
#TODO the noun libraries could be an asset or a plural need to add logic to deal with
# determining which one it is. for now we are going to assume that it is always a
# plural
# If name is not set, set it to the current directory's basename
if name == "." and dir_type in DIR_TYPES:
name = os.path.basename(os.path.realpath("."))
set_init_values(
args,
noun,
)
plural_noun_list = NOUN_PLURALS + [
"hdlplatforms", "hdltargets", "rccplatforms", "rcctargets",
"platforms", "targets"
]
if noun in plural_noun_list:
(noun, action) = get_noun_from_plural(args, noun,
args.get("scope", None))
ocpiutil.logging.debug('Choose noun: "' + str(noun) +
'" and action: "' + action + '"')
if noun not in [
None, "hdlplatforms", "hdltargets", "rccplatforms",
"rcctargets", "platforms", "targets", "projects"
]:
# pylint:disable=unused-variable
ocpiutil.logging.debug("constructing asset noun: " + noun +
" directory: " + directory + "args : " +
str(args))
my_asset = ocpifactory.AssetFactory.factory(
noun, directory, "", **args)
action = ("my_asset." + action + '("' + args["details"] +
'", ' + str(args["verbose"]) + ')')
# pylint:enable=unused-variable
ocpiutil.logging.debug("running show action: '" + action + "'")
# not using anything that is user defined so eval is fine
# pylint:disable=eval-used
eval(action)
# pylint:enable=eval-used
except ocpiutil.OCPIException as ex:
ocpiutil.logging.error(ex)
def main():
"""
Function that is called if this module is called as a mian function
"""
args = parse_cl_vars()
try:
cur_dir = args['cur_dir']
with ocpiutil.cd(cur_dir):
cur_dir_basename = os.path.basename(os.path.realpath("."))
name = args['name']
# Now that we have grabbed name, delete it from the args that will be passed into the
# AssetFactory because name is explicitly passed to AssetFactory as a separate argument
del args['name']
# From the current directory and directory-modifier options, determine
# the directory to operate from for this ocpidev command
directory = ocpiutil.get_ocpidev_working_dir(
origin_path=".",
noun=args.get("noun", ""),
name=name,
library=args['library'],
hdl_library=args['hdl_library'],
hdl_platform=args['hdl_plat_dir'])
ocpiutil.logging.debug('Choose directory "' + directory +
'" to operate in')
dir_type = ocpiutil.get_dirtype(directory)
# Check dir_type for errors:
# If there is no noun, and the working directory is not a supported type
if args['noun'] is None and dir_type not in DIR_TYPES:
raise ocpiutil.OCPIException('Invalid directory type "' +
str(dir_type) +
'" Valid directory types are: ' +
", ".join(DIR_TYPES))
# If name is not set, set it to the current directory's basename
if name == "." and dir_type in DIR_TYPES:
name = cur_dir_basename
# Initialize settings to be used by Asset classes
set_init_values(args, dir_type)
# Check worker authoring model and strip authoring model for Worker construction
if args['noun'] == "worker" or (args['noun'] is None
and dir_type == "worker"):
if not name.endswith(".hdl"):
ocpiutil.logging.warning(
"Can only show utilization for workers of authoring " +
"model 'hdl'.")
sys.exit()
name = name.rsplit('.', 1)[0]
ocpiutil.logging.debug(
"Creating asset object with the following \nname: " +
str(name) + "\n" + "directory: " + str(directory) +
"\nargs: " + str(args))
my_asset = ocpifactory.AssetFactory.factory(
dir_type, directory, name, **args)
my_asset.show_utilization()
except ocpiutil.OCPIException as ex:
ocpiutil.logging.error(ex)
# Construct error message and exit
if args['noun'] is not None:
my_noun = '' if args['noun'] is None else ' "' + args['noun'] + '"'
my_name = '' if name == "." else ' named "' + name + '"'
my_dir = ' in directory "' + args['cur_dir'] + '"'
ocpiutil.logging.error("Unable to show utilization for " +
my_noun + my_name + my_dir +
" due to previous errors")
else:
ocpiutil.logging.error(
"Unable to show utilization due to previous errors.")
sys.exit(1)
