def __init_package_id(self):
"""
Get the Package Name of the project containing 'self.directory'.
"""
# From the project top, probe the Makefile for the projectpackage
# which is printed in cdk/include/project.mk in the projectpackage rule
# if ShellProjectVars is defined
project_package = None
# If the project-package-id file exists, set package-id to its contents
if os.path.isfile(self.directory + "/project-package-id"):
with open(self.directory + "/project-package-id",
"r") as package_id_file:
self.__is_exported = True
project_package = package_id_file.read().strip()
logging.debug("Read Project-ID '" + project_package +
"' from file: " + self.directory +
"/project-package-id")
# Otherwise, ask Makefile at the project top for the ProjectPackage
if project_package is None or project_package == "":
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectpackage ShellProjectVars=1",
verbose=True)
if (not project_vars is None and 'ProjectPackage' in project_vars
and len(project_vars['ProjectPackage']) > 0):
# There is only one value associated with ProjectPackage, so get element 0
project_package = project_vars['ProjectPackage'][0]
else:
raise ocpiutil.OCPIException(
"Could not determine Package-ID of project \"" +
self.directory + "\".")
self.package_id = project_package
def get_valid_tests_workers(self):
"""
Probe make in order to determine the list of active tests in the library
"""
# If this function has alredy been called dont call make again because its very expencive
if self.tests_names is not None and self.wkr_names is not None:
return (self.tests_names, self.wkr_names)
ret_tests = []
ret_wkrs = []
ocpiutil.logging.debug("Getting valid tests from: " + self.directory +
"/Makefile")
make_dict = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="ShellLibraryVars=1 showlib",
verbose=True)
make_tests = make_dict["Tests"]
make_wkrs = make_dict["Workers"]
for name in make_tests:
if name != "":
ret_tests.append(self.directory + "/" + name)
for name in make_wkrs:
if name.endswith((".rcc", ".rcc/", ".hdl", ".hdl/")):
ret_wkrs.append(self.directory + "/" + name)
self.tests_names = ret_tests
self.wkr_names = ret_wkrs
return (ret_tests, ret_wkrs)
def get_make_vars_rcc_targets():
"""
Get make variables from rcc-targets.mk
Dictionary key examples are:
RccAllPlatforms, RccPlatforms, RccAllTargets, RccTargets
"""
return set_vars_from_make(
os.environ["OCPI_CDK_DIR"] + "/include/rcc/rcc-targets.mk",
"ShellRccTargetsVars=1", "verbose")
def get_valid_assemblies(self):
"""
Probes make in order to determine the list of active assemblies in the
assemblies collection
"""
assembs_list = []
ocpiutil.logging.debug("Getting valid assemblies from: " + self.directory + "/Makefile")
make_assembs = ocpiutil.set_vars_from_make(mk_file=self.directory + "/Makefile",
mk_arg="ShellAssembliesVars=1 showassemblies",
verbose=True)["Assemblies"]
# Collect list of assembly directories
for name in make_assembs:
assembs_list.append(self.directory + "/" + name)
return assembs_list
def __init_package_id(self):
"""
Determine the Package id based on the libary or project that the Worker resides in. only
a component will reside at the top level of a project.
"""
parent_dir = self.directory + "/../"
if ocpiutil.get_dirtype(parent_dir) == "library":
ret_val = ocpiutil.set_vars_from_make(
mk_file=parent_dir + "/Makefile",
mk_arg="showpackage ShellLibraryVars=1",
verbose=True)["Package"][0]
elif ocpiutil.get_dirtype(parent_dir) == "project":
ret_val = ocpiutil.set_vars_from_make(
mk_file=parent_dir + "/Makefile",
mk_arg="projectpackage ShellProjectVars=1",
verbose=True)["ProjectPackage"][0]
elif ocpiutil.get_dirtype(parent_dir) == "hdl-platforms":
ret_val = "N/A"
else:
raise ocpiutil.OCPIException(
"Could not determine Package-ID of component dirtype of " +
"parent directory: " + parent_dir + " dirtype: " +
str(ocpiutil.get_dirtype(parent_dir)))
return ret_val
def get_valid_platforms(self):
"""
Probes make in order to determine the list of active platforms in the
platforms collection
"""
platform_list = []
logging.debug("Getting valid platforms from: " + self.directory +
"/Makefile")
make_platforms = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="ShellPlatformsVars=1 showplatforms",
verbose=True)["HdlPlatforms"]
# Collect list of assembly directories
for name in make_platforms:
platform_list.append(self.directory + "/" + name)
return platform_list
def show_libraries(self, details, verbose, **kwargs):
"""
Print out all the libraries that are in this project in the format specified by details
(simple, table, or json)
"""
json_dict = {}
project_dict = {}
libraries_dict = {}
for lib_directory in self.get_valid_libraries():
lib_dict = {}
lib_package = Library.get_package_id(lib_directory)
lib_dict["package"] = lib_package
lib_dict["directory"] = lib_directory
# in case two or more libraries have the same package id we update the key to end
# with a number
i = 1
while lib_package in libraries_dict:
lib_package += ":" + str(i)
i += 1
libraries_dict[lib_package] = lib_dict
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectdeps ShellProjectVars=1",
verbose=True)
project_dict["dependencies"] = project_vars['ProjectDependencies']
project_dict["directory"] = self.directory
project_dict["libraries"] = libraries_dict
project_dict["package"] = self.package_id
json_dict["project"] = project_dict
if details == "simple":
lib_dict = json_dict["project"]["libraries"]
for lib in lib_dict:
print("Library: " + lib_dict[lib]["directory"])
elif details == "table":
rows = [["Library Directories"]]
lib_dict = json_dict["project"]["libraries"]
for lib in lib_dict:
rows.append([lib_dict[lib]["directory"]])
ocpiutil.print_table(rows, underline="-")
else:
json.dump(json_dict, sys.stdout)
print()
def _collect_very_verbose_dict(self):
"""
Generate a dictonary with all the information about a project with verbocity level 2
"""
have_any_tests = False
have_any_wkrs = False
have_any_comps = False
proj_dict = {}
top_dict = {}
top_comp_dict = {}
libraries_dict, have_any_tests, have_any_wkrs, have_any_comps = (
self._get_very_verbose_libs_dict())
for comp in self.comp_list:
top_comp_dict[comp.name] = comp.directory
top_dict["components"] = top_comp_dict
if os.path.isdir(self.directory + "/applications"):
apps_dict = {}
for app_col in self.apps_col_list:
for app in app_col.apps_list:
apps_dict[app.name] = app.directory
top_dict["applications"] = apps_dict
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectdeps ShellProjectVars=1",
verbose=True)
prims_dict = self._get_very_verbose_prims_dict()
if prims_dict:
top_dict["primitives"] = prims_dict
assys_dict = self._get_very_verbose_assemblies_dict()
if assys_dict:
top_dict["assemblies"] = assys_dict
proj_depends = project_vars['ProjectDependencies']
if not proj_depends:
proj_depends.append("None")
top_dict["dependencies"] = proj_depends
top_dict["directory"] = self.directory
top_dict["libraries"] = libraries_dict
top_dict["platforms"] = self._collect_plats_dict()
top_dict["package"] = self.package_id
proj_dict["project"] = top_dict
return proj_dict, have_any_tests, have_any_wkrs, have_any_comps
def get_package_id_wkrs_tests(self, directory='.'):
"""
Return the package id of the Library from the make variable that is returned
"""
lib_vars = ocpiutil.set_vars_from_make(
mk_file=directory + "/Makefile",
mk_arg="ShellLibraryVars=1 showlib",
verbose=True)
ret_package = "".join(lib_vars['Package'])
make_wkrs = lib_vars['Workers'] if lib_vars['Workers'] != [''] else []
make_tests = lib_vars['Tests'] if lib_vars['Tests'] != [''] else []
ret_tests = []
ret_wkrs = []
for name in make_tests:
if name != "":
ret_tests.append(self.directory + "/" + name)
for name in make_wkrs:
if name.endswith((".rcc", ".rcc/", ".hdl", ".hdl/")):
ret_wkrs.append(self.directory + "/" + name)
return ret_package, ret_tests, ret_wkrs
def __init__(self,
name,
target,
exactpart,
directory,
built=False,
package_id=None):
"""
HdlPlatform constructor
"""
super().__init__(directory, name)
self.target = target
self.exactpart = exactpart
self.built = built
self.dir = ocpiutil.rchop(directory, "/lib")
if self.dir and not package_id:
self.package_id = ocpiutil.set_vars_from_make(
self.dir + "/Makefile", "ShellHdlPlatformVars=1 showpackage",
"verbose")["Package"][0]
else:
self.package_id = ""
def _show_non_verbose(self, details, **kwargs):
"""
show all the information about a project with level 1 of verbocity in the format specified
by details (simple, table, or json)
"""
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectdeps ShellProjectVars=1",
verbose=True)
#TODO output the project's registry here too
proj_depends = project_vars['ProjectDependencies']
if not proj_depends:
proj_depends.append("None")
proj_dict = {
'project': {
'directory': self.directory,
'package': self.package_id,
'dependencies': proj_depends
}
}
if details == "simple":
print("Project Directory: " + proj_dict["project"]["directory"])
print("Package-ID: " + proj_dict["project"]["package"])
print("Project Dependencies: " +
", ".join(proj_dict["project"]["dependencies"]))
elif details == "table":
rows = [[
"Project Directory", "Package-ID", "Project Dependencies"
]]
rows.append([
proj_dict["project"]["directory"],
proj_dict["project"]["package"],
", ".join(proj_dict["project"]["dependencies"])
])
ocpiutil.print_table(rows, underline="-")
else:
json.dump(proj_dict, sys.stdout)
print()
def _collect_verbose_dict(self):
"""
Generate a dictonary that contains all the information about a project with verbocity
level 1
"""
proj_dict = {}
top_dict = {}
libraries_dict = {}
top_comp_dict = {}
for comp in self.comp_list:
top_comp_dict[comp.name] = comp.directory
top_dict["components"] = top_comp_dict
for lib in self.get_valid_libraries():
lib_package = Library.get_package_id(lib)
i = 1
while lib_package in libraries_dict:
lib_package += ":" + str(i)
i += 1
libraries_dict[lib_package] = lib
if os.path.isdir(self.directory + "/applications"):
apps_dict = {}
for app_col in self.apps_col_list:
for app in app_col.apps_list:
apps_dict[app.name] = app.directory
top_dict["applications"] = apps_dict
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectdeps ShellProjectVars=1",
verbose=True)
proj_depends = project_vars['ProjectDependencies']
if not proj_depends:
proj_depends.append("None")
top_dict["dependencies"] = proj_depends
top_dict["directory"] = self.directory
top_dict["libraries"] = libraries_dict
top_dict["platforms"] = self._collect_plats_dict()
top_dict["package"] = self.package_id
proj_dict["project"] = top_dict
return proj_dict
def get_make_vars(self):
"""
Collect the list of build configurations and package id for this Platform Worker.
"""
# Get the list of Configurations from make
logging.debug("Get the list of platform Configurations from make")
try:
plat_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="ShellHdlPlatformVars=1 showinfo",
verbose=False)
except ocpiutil.OCPIException:
# if make shoots out an error asume configs are blank
plat_vars = {"Configurations": "", "Package": "N/A"}
if "Configurations" not in plat_vars:
raise ocpiutil.OCPIException(
"Could not get list of HDL Platform Configurations " +
"from \"" + self.directory + "/Makefile\"")
self.package_id = plat_vars["Package"]
# This should be a list of Configuration NAMES
config_list = plat_vars["Configurations"]
return config_list
def get_show_test_dict(self):
"""
Generate the dictionary that is used to show all the tests in this project
"""
json_dict = {}
project_dict = {}
libraries_dict = {}
for lib in self.lib_list:
# pylint:disable=unused-variable
valid_tests, valid_workers = lib.get_valid_tests_workers()
# pylint:disable=unused-variable
if valid_tests:
lib_dict = {}
lib_package = lib.package_id
lib_dict["package"] = lib_package
lib_dict["directory"] = lib.directory
lib_dict["tests"] = {
os.path.basename(test.rstrip('/')): test
for test in valid_tests
}
# in case two or more libraries have the same package id we update the key to end
# with a number
i = 1
while lib_package in libraries_dict:
lib_package += ":" + str(i)
i += 1
libraries_dict[lib_package] = lib_dict
project_vars = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/Makefile",
mk_arg="projectdeps ShellProjectVars=1",
verbose=True)
project_dict["dependencies"] = project_vars['ProjectDependencies']
project_dict["directory"] = self.directory
project_dict["libraries"] = libraries_dict
project_dict["package"] = self.package_id
json_dict["project"] = project_dict
return json_dict
def get_project_package(origin_path="."):
"""
Get the Package Name of the project containing 'origin_path'.
"""
path_to_project = get_path_to_project_top(origin_path)
if path_to_project is None:
logging.debug("Path \"" + str(origin_path) +
"\" is not inside a project")
return None
# From the project top, probe the Makefile for the projectpackage
# which is printed in cdk/include/project.mk in the projectpackage rule
# if ShellProjectVars is defined
with cd(path_to_project):
project_package = None
# If the project-package-id file exists, set package-id to its contents
if os.path.isfile(path_to_project + "/project-package-id"):
with open(path_to_project + "/project-package-id",
"r") as package_id_file:
project_package = package_id_file.read().strip()
logging.debug("Read Project-ID '" + project_package +
"' from file: " + path_to_project +
"/project-package-id")
# Otherwise, ask Makefile at the project top for the ProjectPackage
if project_package is None or project_package == "":
project_vars = set_vars_from_make(
"Makefile", "projectpackage ShellProjectVars=1", "verbose")
if (not project_vars is None and 'ProjectPackage' in project_vars
and len(project_vars['ProjectPackage']) > 0):
# There is only one value associated with ProjectPackage, so get element 0
project_package = project_vars['ProjectPackage'][0]
else:
logging.error("Could not determine Package-ID of project.")
return None
return project_package
def collect_container_info(self, platforms):
"""
Determine which containers exist that support the provided platforms.
Each container will have a simple XML name and a full/implementation name.
Each implementation is bound to a platform and platform configuration.
Return the information in a dictionary with format:
container_impl_dict =
{
xml/simple-name : {
impl-name : HdlPlatform
}
}
"""
container_impl_dict = {}
# Loop through the provided platforms, and determine what containers,
# container implementations, and their corresponding platforms are
# available to this assembly.
for plat in platforms:
plat_string = "HdlPlatform=" + plat.name
# NOTE: We need to call make with a single HdlPlatform set because otherwise
# hdl-pre calls multiple sub-make commands and causes complications
assemb_vars = ocpiutil.set_vars_from_make(mk_file=self.directory + "/Makefile",
mk_arg=plat_string +
" shellhdlassemblyvars " +
"ShellHdlAssemblyVars=1",
verbose=True)
if "Containers" not in assemb_vars:
raise ocpiutil.OCPIException("Could not get list of HDL containers from " +
"directory\"" + self.directory + "\"")
# We get a top-level list of containers (e.g. container XMLs)
for cname in assemb_vars['Containers']:
# Get the list of container IMPLEMENTATIONS for this container XML name
container_impls_str = "HdlContainerImpls_" + cname
# Each container will have a dictionary for its implementations
# Each dict will map implementation-name to supported platform
if cname not in container_impl_dict:
# initialize implementation dict to {} for this container
container_impl_dict[cname] = {}
# Ensure that the list of container implementations is found
if container_impls_str in assemb_vars:
# For each container implementation, determine the corresponding HDL platform
for impl in assemb_vars[container_impls_str]:
container_plat = "HdlPlatform_" + impl
# Construct this implementation's platform and add the mapping to the
# implementation dictionary
plat_obj = hdltargets.HdlToolFactory.factory("hdlplatform",
assemb_vars[container_plat][0])
container_impl_dict[cname][impl] = plat_obj
container_impls_str = "HdlBaseContainerImpls"
# Need to do one more pass to collect information about the base container (if used)
# and collect its implementations as well
if container_impls_str in assemb_vars and assemb_vars['HdlBaseContainerImpls']:
if "base" not in container_impl_dict:
# initialize the implementation dict to {} for the base container
container_impl_dict["base"] = {}
# For each implementation of the base container, determine the corresponding
# HDL platform
for cname in assemb_vars[container_impls_str]:
# Construct this implementation's platform and add the mapping to the
# implementation dictionary
container_plat = "HdlPlatform_" + cname
plat_obj = hdltargets.HdlToolFactory.factory("hdlplatform",
assemb_vars[container_plat][0])
container_impl_dict["base"][cname] = plat_obj
return container_impl_dict
def __parse_hdltargets_from_make(cls):
"""
Ask make for the HDL target/platform/toolset information, and parse it into the
__tool_dict, __tgt_dict and __plat_dict dictionaries. Other functions in this
class will use those dictionaries to construct HdlTarget/Platform/ToolSet intances
"""
if cls.__mk_dict == {}:
# Ask make for tool/tgt/plat info which will be reorganized/parsed below
cls.__mk_dict = ocpiutil.set_vars_from_make(
os.environ["OCPI_CDK_DIR"] + "/include/hdl/hdl-targets.mk",
"ShellHdlTargetsVars=1 " + "ShellGlobalProjectsVars=1",
"verbose")
# Top targets are general groups that likely contain multiple child targets/families
if 'HdlTopTargets' in cls.__mk_dict:
# we call TopTargets "vendors" because that is a more readable term
for vendor in cls.__mk_dict['HdlTopTargets']:
# Each vendor may have a list of associated targets. We assign those
# to the "families" list
if 'HdlTargets_' + vendor in cls.__mk_dict:
families = cls.__mk_dict['HdlTargets_' + vendor]
else:
# if there is no list of targets for the vendor, then the vendor name is
# itself the target to use
families = [vendor]
# for each family, create an entry in the class' tgt_dict. set the vendor, list
# of associated parts, the toolset, and some other metadata
for family in families:
cls.__tgt_dict[family] = {}
cls.__tgt_dict[family]["vendor"] = vendor
# A family will have a list of associated parts, so get them and
# add them to the tgt_dict under the "parts" entry for this family
if 'HdlTargets_' + family in cls.__mk_dict:
cls.__tgt_dict[family]["parts"] = cls.__mk_dict[
'HdlTargets_' + family]
else:
# if there is no parts list, then the family itself is the part
cls.__tgt_dict[family]["parts"] = [family]
# There must be a toolset associated with each family, so get it and add
# it to the tgt_dict under the family's "toolset" entry
toolname = cls.__mk_dict['HdlToolSet_' + family][0]
cls.__tgt_dict[family]["toolset"] = toolname
# Add this tool to the separate tool_dict along with its associated title,
# and whether or not it is a simulator tool
if toolname not in cls.__tool_dict:
cls.__tool_dict[toolname] = {}
# Get the title for this tool via HdlToolName
if 'HdlToolName_' + toolname in cls.__mk_dict:
cls.__tool_dict[toolname]["title"] = \
cls.__mk_dict['HdlToolName_' + toolname][0]
# Determine if this tool is one of the HdlSimTools. Set the is_simtool
# entry in __tool_dict accordingly
is_simtool = ('HdlSimTools' in cls.__mk_dict and family
in cls.__mk_dict['HdlSimTools'])
cls.__tool_dict[toolname]["is_simtool"] = is_simtool
# For each HdlPlatform, add an entry to __plat_dict including the platform's exact part,
# its HDL target name, and whether it is built
if 'HdlAllPlatforms' in cls.__mk_dict:
for platname in cls.__mk_dict['HdlAllPlatforms']:
cls.__plat_dict[platname] = {}
exactpart = cls.__mk_dict['HdlPart_' + platname][0]
cls.__plat_dict[platname]['exactpart'] = exactpart
cls.__plat_dict[platname]['targetname'] = cls.__mk_dict[
'HdlFamily_' + exactpart][0]
cls.__plat_dict[platname]['built'] = platname in cls.__mk_dict[
'HdlBuiltPlatforms']
def get_ocpigen_metadata(self, xml_file):
"""
Ask ocpigen (the code generator) to parse the worker(OWD) or component(OCS) xml file and
spit out an artifact xml that this fuction parses into class variables.
property_list - list of every property, each property in this list will be a dictionary of
all the xml attributes asscoiated with it from the artifact xml
port_list - list of every port each port in this list will be a dictionary of
all the xml attributes asscoiated with it from the artifact xml some of
which are unused
slave_list - list of every slave worker's name expected to be blank for an OCS
Function attribtes:
xml_file - the file to have ocpigen parse
"""
#get list of locations to look for include xmls from make
if ocpiutil.get_dirtype(self.directory + "/../") == "library":
xml_dirs = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/../" + "/Makefile",
mk_arg="showincludes ShellLibraryVars=1",
verbose=True)["XmlIncludeDirsInternal"]
elif ocpiutil.get_dirtype(self.directory + "/../") == "project":
xml_dirs = ocpiutil.set_vars_from_make(
mk_file=self.directory + "/../" + "/Makefile",
mk_arg="projectincludes ShellProjectVars=1",
verbose=True)["XmlIncludeDirsInternal"]
#call ocpigen -G
ocpigen_cmd = [
"ocpigen", "-G", "-O", "none", "-V", "none", "-H", "none"
]
for inc_dir in xml_dirs:
ocpigen_cmd.append("-I")
ocpigen_cmd.append(inc_dir)
ocpigen_cmd.append(xml_file)
ocpiutil.logging.debug("running ocpigen cmd: " + str(ocpigen_cmd))
old_log_level = os.environ.get("OCPI_LOG_LEVEL", "0")
os.environ["OCPI_LOG_LEVEL"] = "0"
comp_xml = subprocess.Popen(ocpigen_cmd,
stdout=subprocess.PIPE).communicate()[0]
os.environ["OCPI_LOG_LEVEL"] = old_log_level
#put xml ouput file into an ElementTree object
ocpiutil.logging.debug("Component Artifact XML from ocpigen: \n" +
str(comp_xml))
try:
parsed_xml = ET.fromstring(comp_xml)
except ET.ParseError:
raise ocpiutil.OCPIException(
"Error with xml file from ocpigen.\n\nocpigen command: " +
str(ocpigen_cmd) + "\n\nxml output: \n" + str(comp_xml))
self.property_list = []
self.port_list = []
self.slave_list = []
#set up self.property_list from the xml
for props in parsed_xml.findall("property"):
temp_dict = props.attrib
enum = temp_dict.get("enums")
if enum:
temp_dict["enums"] = enum.split(",")
if props.attrib.get("type") == "Struct":
temp_dict["Struct"] = self.get_struct_dict_from_xml(
props.findall("member"))
self.property_list.append(temp_dict)
#set up self.port_list from this dict
for port in parsed_xml.findall("port"):
port_details = port.attrib
for child in port:
if child.tag == "protocol":
port_details["protocol"] = child.attrib.get(
"padding", "N/A")
self.port_list.append(port_details)
#set up self.slave_list from the xml
for slave in parsed_xml.findall("slave"):
self.slave_list.append(slave.attrib["worker"])
def get_package_id(directory='.'):
lib_vars = ocpiutil.set_vars_from_make(
mk_file=directory + "/Makefile",
mk_arg="ShellLibraryVars=1 showlib",
verbose=True)
return "".join(lib_vars['Package'])
