def __init_package_id(self): """ Get the Package Name of the project containing 'self.directory'. """ # From the project top, probe the Makefile for the projectpackage # which is printed in cdk/include/project.mk in the projectpackage rule # if ShellProjectVars is defined project_package = None # If the project-package-id file exists, set package-id to its contents if os.path.isfile(self.directory + "/project-package-id"): with open(self.directory + "/project-package-id", "r") as package_id_file: self.__is_exported = True project_package = package_id_file.read().strip() logging.debug("Read Project-ID '" + project_package + "' from file: " + self.directory + "/project-package-id") # Otherwise, ask Makefile at the project top for the ProjectPackage if project_package is None or project_package == "": project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectpackage ShellProjectVars=1", verbose=True) if (not project_vars is None and 'ProjectPackage' in project_vars and len(project_vars['ProjectPackage']) > 0): # There is only one value associated with ProjectPackage, so get element 0 project_package = project_vars['ProjectPackage'][0] else: raise ocpiutil.OCPIException( "Could not determine Package-ID of project \"" + self.directory + "\".") self.package_id = project_package
def get_valid_tests_workers(self): """ Probe make in order to determine the list of active tests in the library """ # If this function has alredy been called dont call make again because its very expencive if self.tests_names is not None and self.wkr_names is not None: return (self.tests_names, self.wkr_names) ret_tests = [] ret_wkrs = [] ocpiutil.logging.debug("Getting valid tests from: " + self.directory + "/Makefile") make_dict = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="ShellLibraryVars=1 showlib", verbose=True) make_tests = make_dict["Tests"] make_wkrs = make_dict["Workers"] for name in make_tests: if name != "": ret_tests.append(self.directory + "/" + name) for name in make_wkrs: if name.endswith((".rcc", ".rcc/", ".hdl", ".hdl/")): ret_wkrs.append(self.directory + "/" + name) self.tests_names = ret_tests self.wkr_names = ret_wkrs return (ret_tests, ret_wkrs)
def get_make_vars_rcc_targets(): """ Get make variables from rcc-targets.mk Dictionary key examples are: RccAllPlatforms, RccPlatforms, RccAllTargets, RccTargets """ return set_vars_from_make( os.environ["OCPI_CDK_DIR"] + "/include/rcc/rcc-targets.mk", "ShellRccTargetsVars=1", "verbose")
def get_valid_assemblies(self): """ Probes make in order to determine the list of active assemblies in the assemblies collection """ assembs_list = [] ocpiutil.logging.debug("Getting valid assemblies from: " + self.directory + "/Makefile") make_assembs = ocpiutil.set_vars_from_make(mk_file=self.directory + "/Makefile", mk_arg="ShellAssembliesVars=1 showassemblies", verbose=True)["Assemblies"] # Collect list of assembly directories for name in make_assembs: assembs_list.append(self.directory + "/" + name) return assembs_list
def __init_package_id(self): """ Determine the Package id based on the libary or project that the Worker resides in. only a component will reside at the top level of a project. """ parent_dir = self.directory + "/../" if ocpiutil.get_dirtype(parent_dir) == "library": ret_val = ocpiutil.set_vars_from_make( mk_file=parent_dir + "/Makefile", mk_arg="showpackage ShellLibraryVars=1", verbose=True)["Package"][0] elif ocpiutil.get_dirtype(parent_dir) == "project": ret_val = ocpiutil.set_vars_from_make( mk_file=parent_dir + "/Makefile", mk_arg="projectpackage ShellProjectVars=1", verbose=True)["ProjectPackage"][0] elif ocpiutil.get_dirtype(parent_dir) == "hdl-platforms": ret_val = "N/A" else: raise ocpiutil.OCPIException( "Could not determine Package-ID of component dirtype of " + "parent directory: " + parent_dir + " dirtype: " + str(ocpiutil.get_dirtype(parent_dir))) return ret_val
def get_valid_platforms(self): """ Probes make in order to determine the list of active platforms in the platforms collection """ platform_list = [] logging.debug("Getting valid platforms from: " + self.directory + "/Makefile") make_platforms = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="ShellPlatformsVars=1 showplatforms", verbose=True)["HdlPlatforms"] # Collect list of assembly directories for name in make_platforms: platform_list.append(self.directory + "/" + name) return platform_list
def show_libraries(self, details, verbose, **kwargs): """ Print out all the libraries that are in this project in the format specified by details (simple, table, or json) """ json_dict = {} project_dict = {} libraries_dict = {} for lib_directory in self.get_valid_libraries(): lib_dict = {} lib_package = Library.get_package_id(lib_directory) lib_dict["package"] = lib_package lib_dict["directory"] = lib_directory # in case two or more libraries have the same package id we update the key to end # with a number i = 1 while lib_package in libraries_dict: lib_package += ":" + str(i) i += 1 libraries_dict[lib_package] = lib_dict project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectdeps ShellProjectVars=1", verbose=True) project_dict["dependencies"] = project_vars['ProjectDependencies'] project_dict["directory"] = self.directory project_dict["libraries"] = libraries_dict project_dict["package"] = self.package_id json_dict["project"] = project_dict if details == "simple": lib_dict = json_dict["project"]["libraries"] for lib in lib_dict: print("Library: " + lib_dict[lib]["directory"]) elif details == "table": rows = [["Library Directories"]] lib_dict = json_dict["project"]["libraries"] for lib in lib_dict: rows.append([lib_dict[lib]["directory"]]) ocpiutil.print_table(rows, underline="-") else: json.dump(json_dict, sys.stdout) print()
def _collect_very_verbose_dict(self): """ Generate a dictonary with all the information about a project with verbocity level 2 """ have_any_tests = False have_any_wkrs = False have_any_comps = False proj_dict = {} top_dict = {} top_comp_dict = {} libraries_dict, have_any_tests, have_any_wkrs, have_any_comps = ( self._get_very_verbose_libs_dict()) for comp in self.comp_list: top_comp_dict[comp.name] = comp.directory top_dict["components"] = top_comp_dict if os.path.isdir(self.directory + "/applications"): apps_dict = {} for app_col in self.apps_col_list: for app in app_col.apps_list: apps_dict[app.name] = app.directory top_dict["applications"] = apps_dict project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectdeps ShellProjectVars=1", verbose=True) prims_dict = self._get_very_verbose_prims_dict() if prims_dict: top_dict["primitives"] = prims_dict assys_dict = self._get_very_verbose_assemblies_dict() if assys_dict: top_dict["assemblies"] = assys_dict proj_depends = project_vars['ProjectDependencies'] if not proj_depends: proj_depends.append("None") top_dict["dependencies"] = proj_depends top_dict["directory"] = self.directory top_dict["libraries"] = libraries_dict top_dict["platforms"] = self._collect_plats_dict() top_dict["package"] = self.package_id proj_dict["project"] = top_dict return proj_dict, have_any_tests, have_any_wkrs, have_any_comps
def get_package_id_wkrs_tests(self, directory='.'): """ Return the package id of the Library from the make variable that is returned """ lib_vars = ocpiutil.set_vars_from_make( mk_file=directory + "/Makefile", mk_arg="ShellLibraryVars=1 showlib", verbose=True) ret_package = "".join(lib_vars['Package']) make_wkrs = lib_vars['Workers'] if lib_vars['Workers'] != [''] else [] make_tests = lib_vars['Tests'] if lib_vars['Tests'] != [''] else [] ret_tests = [] ret_wkrs = [] for name in make_tests: if name != "": ret_tests.append(self.directory + "/" + name) for name in make_wkrs: if name.endswith((".rcc", ".rcc/", ".hdl", ".hdl/")): ret_wkrs.append(self.directory + "/" + name) return ret_package, ret_tests, ret_wkrs
def __init__(self, name, target, exactpart, directory, built=False, package_id=None): """ HdlPlatform constructor """ super().__init__(directory, name) self.target = target self.exactpart = exactpart self.built = built self.dir = ocpiutil.rchop(directory, "/lib") if self.dir and not package_id: self.package_id = ocpiutil.set_vars_from_make( self.dir + "/Makefile", "ShellHdlPlatformVars=1 showpackage", "verbose")["Package"][0] else: self.package_id = ""
def _show_non_verbose(self, details, **kwargs): """ show all the information about a project with level 1 of verbocity in the format specified by details (simple, table, or json) """ project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectdeps ShellProjectVars=1", verbose=True) #TODO output the project's registry here too proj_depends = project_vars['ProjectDependencies'] if not proj_depends: proj_depends.append("None") proj_dict = { 'project': { 'directory': self.directory, 'package': self.package_id, 'dependencies': proj_depends } } if details == "simple": print("Project Directory: " + proj_dict["project"]["directory"]) print("Package-ID: " + proj_dict["project"]["package"]) print("Project Dependencies: " + ", ".join(proj_dict["project"]["dependencies"])) elif details == "table": rows = [[ "Project Directory", "Package-ID", "Project Dependencies" ]] rows.append([ proj_dict["project"]["directory"], proj_dict["project"]["package"], ", ".join(proj_dict["project"]["dependencies"]) ]) ocpiutil.print_table(rows, underline="-") else: json.dump(proj_dict, sys.stdout) print()
def _collect_verbose_dict(self): """ Generate a dictonary that contains all the information about a project with verbocity level 1 """ proj_dict = {} top_dict = {} libraries_dict = {} top_comp_dict = {} for comp in self.comp_list: top_comp_dict[comp.name] = comp.directory top_dict["components"] = top_comp_dict for lib in self.get_valid_libraries(): lib_package = Library.get_package_id(lib) i = 1 while lib_package in libraries_dict: lib_package += ":" + str(i) i += 1 libraries_dict[lib_package] = lib if os.path.isdir(self.directory + "/applications"): apps_dict = {} for app_col in self.apps_col_list: for app in app_col.apps_list: apps_dict[app.name] = app.directory top_dict["applications"] = apps_dict project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectdeps ShellProjectVars=1", verbose=True) proj_depends = project_vars['ProjectDependencies'] if not proj_depends: proj_depends.append("None") top_dict["dependencies"] = proj_depends top_dict["directory"] = self.directory top_dict["libraries"] = libraries_dict top_dict["platforms"] = self._collect_plats_dict() top_dict["package"] = self.package_id proj_dict["project"] = top_dict return proj_dict
def get_make_vars(self): """ Collect the list of build configurations and package id for this Platform Worker. """ # Get the list of Configurations from make logging.debug("Get the list of platform Configurations from make") try: plat_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="ShellHdlPlatformVars=1 showinfo", verbose=False) except ocpiutil.OCPIException: # if make shoots out an error asume configs are blank plat_vars = {"Configurations": "", "Package": "N/A"} if "Configurations" not in plat_vars: raise ocpiutil.OCPIException( "Could not get list of HDL Platform Configurations " + "from \"" + self.directory + "/Makefile\"") self.package_id = plat_vars["Package"] # This should be a list of Configuration NAMES config_list = plat_vars["Configurations"] return config_list
def get_show_test_dict(self): """ Generate the dictionary that is used to show all the tests in this project """ json_dict = {} project_dict = {} libraries_dict = {} for lib in self.lib_list: # pylint:disable=unused-variable valid_tests, valid_workers = lib.get_valid_tests_workers() # pylint:disable=unused-variable if valid_tests: lib_dict = {} lib_package = lib.package_id lib_dict["package"] = lib_package lib_dict["directory"] = lib.directory lib_dict["tests"] = { os.path.basename(test.rstrip('/')): test for test in valid_tests } # in case two or more libraries have the same package id we update the key to end # with a number i = 1 while lib_package in libraries_dict: lib_package += ":" + str(i) i += 1 libraries_dict[lib_package] = lib_dict project_vars = ocpiutil.set_vars_from_make( mk_file=self.directory + "/Makefile", mk_arg="projectdeps ShellProjectVars=1", verbose=True) project_dict["dependencies"] = project_vars['ProjectDependencies'] project_dict["directory"] = self.directory project_dict["libraries"] = libraries_dict project_dict["package"] = self.package_id json_dict["project"] = project_dict return json_dict
def get_project_package(origin_path="."): """ Get the Package Name of the project containing 'origin_path'. """ path_to_project = get_path_to_project_top(origin_path) if path_to_project is None: logging.debug("Path \"" + str(origin_path) + "\" is not inside a project") return None # From the project top, probe the Makefile for the projectpackage # which is printed in cdk/include/project.mk in the projectpackage rule # if ShellProjectVars is defined with cd(path_to_project): project_package = None # If the project-package-id file exists, set package-id to its contents if os.path.isfile(path_to_project + "/project-package-id"): with open(path_to_project + "/project-package-id", "r") as package_id_file: project_package = package_id_file.read().strip() logging.debug("Read Project-ID '" + project_package + "' from file: " + path_to_project + "/project-package-id") # Otherwise, ask Makefile at the project top for the ProjectPackage if project_package is None or project_package == "": project_vars = set_vars_from_make( "Makefile", "projectpackage ShellProjectVars=1", "verbose") if (not project_vars is None and 'ProjectPackage' in project_vars and len(project_vars['ProjectPackage']) > 0): # There is only one value associated with ProjectPackage, so get element 0 project_package = project_vars['ProjectPackage'][0] else: logging.error("Could not determine Package-ID of project.") return None return project_package
def collect_container_info(self, platforms): """ Determine which containers exist that support the provided platforms. Each container will have a simple XML name and a full/implementation name. Each implementation is bound to a platform and platform configuration. Return the information in a dictionary with format: container_impl_dict = { xml/simple-name : { impl-name : HdlPlatform } } """ container_impl_dict = {} # Loop through the provided platforms, and determine what containers, # container implementations, and their corresponding platforms are # available to this assembly. for plat in platforms: plat_string = "HdlPlatform=" + plat.name # NOTE: We need to call make with a single HdlPlatform set because otherwise # hdl-pre calls multiple sub-make commands and causes complications assemb_vars = ocpiutil.set_vars_from_make(mk_file=self.directory + "/Makefile", mk_arg=plat_string + " shellhdlassemblyvars " + "ShellHdlAssemblyVars=1", verbose=True) if "Containers" not in assemb_vars: raise ocpiutil.OCPIException("Could not get list of HDL containers from " + "directory\"" + self.directory + "\"") # We get a top-level list of containers (e.g. container XMLs) for cname in assemb_vars['Containers']: # Get the list of container IMPLEMENTATIONS for this container XML name container_impls_str = "HdlContainerImpls_" + cname # Each container will have a dictionary for its implementations # Each dict will map implementation-name to supported platform if cname not in container_impl_dict: # initialize implementation dict to {} for this container container_impl_dict[cname] = {} # Ensure that the list of container implementations is found if container_impls_str in assemb_vars: # For each container implementation, determine the corresponding HDL platform for impl in assemb_vars[container_impls_str]: container_plat = "HdlPlatform_" + impl # Construct this implementation's platform and add the mapping to the # implementation dictionary plat_obj = hdltargets.HdlToolFactory.factory("hdlplatform", assemb_vars[container_plat][0]) container_impl_dict[cname][impl] = plat_obj container_impls_str = "HdlBaseContainerImpls" # Need to do one more pass to collect information about the base container (if used) # and collect its implementations as well if container_impls_str in assemb_vars and assemb_vars['HdlBaseContainerImpls']: if "base" not in container_impl_dict: # initialize the implementation dict to {} for the base container container_impl_dict["base"] = {} # For each implementation of the base container, determine the corresponding # HDL platform for cname in assemb_vars[container_impls_str]: # Construct this implementation's platform and add the mapping to the # implementation dictionary container_plat = "HdlPlatform_" + cname plat_obj = hdltargets.HdlToolFactory.factory("hdlplatform", assemb_vars[container_plat][0]) container_impl_dict["base"][cname] = plat_obj return container_impl_dict
def __parse_hdltargets_from_make(cls): """ Ask make for the HDL target/platform/toolset information, and parse it into the __tool_dict, __tgt_dict and __plat_dict dictionaries. Other functions in this class will use those dictionaries to construct HdlTarget/Platform/ToolSet intances """ if cls.__mk_dict == {}: # Ask make for tool/tgt/plat info which will be reorganized/parsed below cls.__mk_dict = ocpiutil.set_vars_from_make( os.environ["OCPI_CDK_DIR"] + "/include/hdl/hdl-targets.mk", "ShellHdlTargetsVars=1 " + "ShellGlobalProjectsVars=1", "verbose") # Top targets are general groups that likely contain multiple child targets/families if 'HdlTopTargets' in cls.__mk_dict: # we call TopTargets "vendors" because that is a more readable term for vendor in cls.__mk_dict['HdlTopTargets']: # Each vendor may have a list of associated targets. We assign those # to the "families" list if 'HdlTargets_' + vendor in cls.__mk_dict: families = cls.__mk_dict['HdlTargets_' + vendor] else: # if there is no list of targets for the vendor, then the vendor name is # itself the target to use families = [vendor] # for each family, create an entry in the class' tgt_dict. set the vendor, list # of associated parts, the toolset, and some other metadata for family in families: cls.__tgt_dict[family] = {} cls.__tgt_dict[family]["vendor"] = vendor # A family will have a list of associated parts, so get them and # add them to the tgt_dict under the "parts" entry for this family if 'HdlTargets_' + family in cls.__mk_dict: cls.__tgt_dict[family]["parts"] = cls.__mk_dict[ 'HdlTargets_' + family] else: # if there is no parts list, then the family itself is the part cls.__tgt_dict[family]["parts"] = [family] # There must be a toolset associated with each family, so get it and add # it to the tgt_dict under the family's "toolset" entry toolname = cls.__mk_dict['HdlToolSet_' + family][0] cls.__tgt_dict[family]["toolset"] = toolname # Add this tool to the separate tool_dict along with its associated title, # and whether or not it is a simulator tool if toolname not in cls.__tool_dict: cls.__tool_dict[toolname] = {} # Get the title for this tool via HdlToolName if 'HdlToolName_' + toolname in cls.__mk_dict: cls.__tool_dict[toolname]["title"] = \ cls.__mk_dict['HdlToolName_' + toolname][0] # Determine if this tool is one of the HdlSimTools. Set the is_simtool # entry in __tool_dict accordingly is_simtool = ('HdlSimTools' in cls.__mk_dict and family in cls.__mk_dict['HdlSimTools']) cls.__tool_dict[toolname]["is_simtool"] = is_simtool # For each HdlPlatform, add an entry to __plat_dict including the platform's exact part, # its HDL target name, and whether it is built if 'HdlAllPlatforms' in cls.__mk_dict: for platname in cls.__mk_dict['HdlAllPlatforms']: cls.__plat_dict[platname] = {} exactpart = cls.__mk_dict['HdlPart_' + platname][0] cls.__plat_dict[platname]['exactpart'] = exactpart cls.__plat_dict[platname]['targetname'] = cls.__mk_dict[ 'HdlFamily_' + exactpart][0] cls.__plat_dict[platname]['built'] = platname in cls.__mk_dict[ 'HdlBuiltPlatforms']
def get_ocpigen_metadata(self, xml_file): """ Ask ocpigen (the code generator) to parse the worker(OWD) or component(OCS) xml file and spit out an artifact xml that this fuction parses into class variables. property_list - list of every property, each property in this list will be a dictionary of all the xml attributes asscoiated with it from the artifact xml port_list - list of every port each port in this list will be a dictionary of all the xml attributes asscoiated with it from the artifact xml some of which are unused slave_list - list of every slave worker's name expected to be blank for an OCS Function attribtes: xml_file - the file to have ocpigen parse """ #get list of locations to look for include xmls from make if ocpiutil.get_dirtype(self.directory + "/../") == "library": xml_dirs = ocpiutil.set_vars_from_make( mk_file=self.directory + "/../" + "/Makefile", mk_arg="showincludes ShellLibraryVars=1", verbose=True)["XmlIncludeDirsInternal"] elif ocpiutil.get_dirtype(self.directory + "/../") == "project": xml_dirs = ocpiutil.set_vars_from_make( mk_file=self.directory + "/../" + "/Makefile", mk_arg="projectincludes ShellProjectVars=1", verbose=True)["XmlIncludeDirsInternal"] #call ocpigen -G ocpigen_cmd = [ "ocpigen", "-G", "-O", "none", "-V", "none", "-H", "none" ] for inc_dir in xml_dirs: ocpigen_cmd.append("-I") ocpigen_cmd.append(inc_dir) ocpigen_cmd.append(xml_file) ocpiutil.logging.debug("running ocpigen cmd: " + str(ocpigen_cmd)) old_log_level = os.environ.get("OCPI_LOG_LEVEL", "0") os.environ["OCPI_LOG_LEVEL"] = "0" comp_xml = subprocess.Popen(ocpigen_cmd, stdout=subprocess.PIPE).communicate()[0] os.environ["OCPI_LOG_LEVEL"] = old_log_level #put xml ouput file into an ElementTree object ocpiutil.logging.debug("Component Artifact XML from ocpigen: \n" + str(comp_xml)) try: parsed_xml = ET.fromstring(comp_xml) except ET.ParseError: raise ocpiutil.OCPIException( "Error with xml file from ocpigen.\n\nocpigen command: " + str(ocpigen_cmd) + "\n\nxml output: \n" + str(comp_xml)) self.property_list = [] self.port_list = [] self.slave_list = [] #set up self.property_list from the xml for props in parsed_xml.findall("property"): temp_dict = props.attrib enum = temp_dict.get("enums") if enum: temp_dict["enums"] = enum.split(",") if props.attrib.get("type") == "Struct": temp_dict["Struct"] = self.get_struct_dict_from_xml( props.findall("member")) self.property_list.append(temp_dict) #set up self.port_list from this dict for port in parsed_xml.findall("port"): port_details = port.attrib for child in port: if child.tag == "protocol": port_details["protocol"] = child.attrib.get( "padding", "N/A") self.port_list.append(port_details) #set up self.slave_list from the xml for slave in parsed_xml.findall("slave"): self.slave_list.append(slave.attrib["worker"])
def get_package_id(directory='.'): lib_vars = ocpiutil.set_vars_from_make( mk_file=directory + "/Makefile", mk_arg="ShellLibraryVars=1 showlib", verbose=True) return "".join(lib_vars['Package'])