def match_regex_get_first_num(target_file, regex):
"""
Match the given regex in the given file (get the first match),
and return the first number in the matched string.
"""
# TODO add doctest
if isinstance(regex, str):
# if regex is a string, compile to regex
regex = re.compile(regex)
elif not isinstance(regex, type(re.compile(''))):
raise OCPIException("Error: regular expression invalid")
# Have to use encoding/errors flags here to avoid failure when file has non-ASCII
with open(target_file, encoding="utf8", errors='ignore') as tgt_file:
for line in tgt_file:
# List of all matches on the line
matches = re.findall(regex, line)
# Were any matches found?
if matches and matches[0]:
# Get the first number in the first match on the line
first_num = first_num_in_str(matches[0])
if first_num != None:
return first_num
# no number/match found
return None
def match_regex(target_file, regex):
"""
Parse the target file for a regex and return the first group/element
of the first match
For example (doctest):
Here, we have a line of text below and we parse this current file
using a regular expression to grab the 'ns' number from the string.
# This text right here contains a number (2.000ns)!!!! It should match the regex
>>> match_regex(os.path.realpath(__file__), "#.*This text right.*\((.*)ns\)!!!!.*")
'2.000'
"""
if isinstance(regex, str):
regex = re.compile(regex)
elif not isinstance(regex, type(re.compile(''))):
raise OCPIException("Error: regular expression invalid")
# Have to use encoding/errors flags here to avoid failure when file has non-ASCII
with open(target_file, encoding="utf8", errors="ignore") as tgt_file:
for line in tgt_file:
# List of all matches on the line
matches = re.findall(regex, line)
# Were any matches found?
if matches and matches[0]:
# Get the first match on the line
match = matches[0]
if isinstance(match, tuple):
# Match might be a tuple (if 1+ groups are in the pattern). In that case
# return the first group in the match tuple
return match[0]
return match
# no match found
return None
def execute_cmd(settings, directory, action=None):
"""
This command is a wrapper around any calls to make in order to encapsulate the use of make to a
minimal number of places. The function contains a hard-coded dictionary of generic settings to
make variables that it uses to construct the call to make at that given directory
"""
settings_dict = {'rcc_plats' : "RccPlatforms",
'hdl_plat_strs' : "HdlPlatforms",
'hdl_tgt_strs' : "HdlTargets",
'only_plats' : "OnlyPlatforms",
'ex_plats' : "ExcludePlatforms",
'keep_sims' : "KeepSimulations",
'acc_errors' : "TestAccumulateErrors",
'view' : "View",
'cases' : "Cases",
'run_before' : "OcpiRunBefore",
'run_after' : "OcpiRunAfter",
'run_arg' : "OcpiRunArgs",
'remote_test_sys' : "OCPI_REMOTE_TEST_SYSTEMS",
'verbose' : "TestVerbose"}
make_list = []
make_list.append("make")
make_list.append("-C")
make_list.append(directory)
debug_string = "make -C " + directory
if action is not None:
make_list.extend(action)
debug_string += " " + ' '.join(action) + " "
logging.debug("settings for make command: " + str(settings.items()))
for setting, value in settings.items():
if isinstance(value, bool):
make_list.append(settings_dict[setting] + '=1')
debug_string += " " + settings_dict[setting] + '=1'
elif isinstance(value, list) or isinstance(value, set):
make_list.append(settings_dict[setting] + '=' + ' '.join(value))
if len(value) > 1:
debug_string += " " + settings_dict[setting] + '="' + ' '.join(value) + '"'
else:
debug_string += " " + settings_dict[setting] + '=' + ' '.join(value)
else:
raise OCPIException("Invalid setting data-type passed to execute_cmd(). Valid data-" +
"types are bool and list")
logging.debug("running make command: " + debug_string)
#shell=True is bad dont set it here running the following command was able to execute
# arbitary code
#ocpidev run test --hdl-platform \$\(./script.temp\)
# all the script would need to do is cat isim then go on its merry way doing whatever it wanted
child = subprocess.Popen(make_list)
child.wait()
return child.returncode
def print_table(self, none_str="N/A"):
"""
Print a table of this Report's data-points. Lead with the headers and present
each data-point's elements as per the order in "ordered_headers".
Let the caller specify a string to use for empty/None entries (Default: N/A)
"""
if self.data_points:
# Fill in empty values in data-points with none_str
for header in self.ordered_headers:
for point in self.data_points:
if header not in point:
point[header] = none_str
# Construct the list of rows. Do so following the ordered_headers order
rows = []
for point in self.data_points:
row = []
for header in self.ordered_headers:
elem = point[header]
if elem is None:
row.append(none_str)
elif isinstance(elem, str):
# element is a string, just append to row
row.append(elem)
elif isinstance(elem, list):
# element is a list, join its contents with comma and append to row
row.append(", ".join(elem))
else:
# Unsupported element type (not a string or list)
try:
row.append(str(elem))
except Exception as ex:
logging.error(ex)
raise OCPIException(
"Element in Report is not a string and could not"
+ " be cast to a string.")
rows.append(row)
# Sort the rows by each header listed in sort_priority. Do this in reverse order
# so the last sort key (and therefore the most significant) is the first element
# in sort_priority
for sort_header in reversed(self.sort_priority):
rows.sort(key=operator.itemgetter(
self.ordered_headers.index(sort_header)))
# insert the headers as row 0
rows.insert(0, self.ordered_headers)
# format the table using a generic function and print
print(format_table(rows, underline="-"))
else:
logging.warning("Not printing table for empty report.")
def __iadd__(self, other):
"""
Adding a Report to this one is the equivalent of adding a Report's data_points
to this Reports data-point list. If this Report's ordered_headers is empty,
use the other Report's ordered_headers.
if not isinstance(other, Report):
"""
if not isinstance(other, Report):
raise OCPIException(
"Can only add another Report object to a Report.")
# Headers are unchanged on addition unless headers are empty in which case,
# user other's headers
if not self.ordered_headers:
self.ordered_headers = other.ordered_headers
self.data_points += other.data_points
return self
def get_paths_through_project_top(origin_path, dest_paths):
"""
origin to each destination going through the project top (and potentially
imports) when possible.
"""
origin_top = get_path_to_project_top(origin_path, False)
origin_top_rel = get_path_to_project_top(origin_path, True)
if origin_top is None:
raise OCPIException("origin_path \"" + str(origin_path) +
"\" is not in a project")
paths_through_top = []
for dest_p in dest_paths:
paths_through_top.append(
get_path_from_given_project_top(origin_top, dest_p,
origin_top_rel))
return paths_through_top
def get_paths_from_project_top(origin_path, dest_paths):
"""
Given an origin and a list of destination paths, return a list of paths
from origin's project top to each destination (potentially through imports).
If a destination path is not in the current or an imported project,
return an absolute path.
"""
origin_top = get_path_to_project_top(origin_path, False)
if origin_top is None:
raise OCPIException("origin_path \"" + str(origin_path) +
"\" is not in a project")
paths_from_top = []
for dest_p in dest_paths:
paths_from_top.append(
get_path_from_given_project_top(origin_top, dest_p))
return paths_from_top
def _get_asset_dir(noun="",
name=".",
library=None,
hdl_library=None,
hdl_platform=None):
"""
Given a noun, a name, and library directives
determine the target asset's directory from a project top.
Basically, the arguments of this function specify/describe an asset in a project.
This function completes that mapping of asset-description -> asset-directory,
and returns the directory of interest relative to the top of the project.
Args:
noun: noun/asset-type of interest. Defaults to blank
name: name of the asset to locate. Basically, what is the name of the asset of type <noun>
library: the library in which the asset of interest lives
hdl_library: the HDL library in which the asset of interest lives (in the hdl/ subtree)
hdl_platform: the HDL platform in which the asset of interest lives.
might itself be the asset
Return:
asset: the project subdirectory containing the asset described via this function's arguments
Notes:
name and noun describe the name and type of asset being described.
The library and platform arguments are location-directives or modifiers
that dictate where to the asset is found.
Examples (doctest):
>>> _get_asset_dir(noun="library", name="components")
'components'
>>> _get_asset_dir(noun="library", name="dsp_comps")
'components/dsp_comps'
>>> _get_asset_dir(noun="library", name="components/dsp_comps")
'components/dsp_comps'
>>> _get_asset_dir(noun="library", name="hdl/devices")
'hdl/devices'
>>> _get_asset_dir(noun="worker", name="complex_mixer.hdl", library="dsp_comps")
'components/dsp_comps/complex_mixer.hdl'
>>> _get_asset_dir(noun="test", name="bias.test", library="components")
'components/bias.test'
>>> _get_asset_dir(noun="test", name="bias", library="components")
'components/bias.test'
>>> _get_asset_dir(noun="worker", name="ad9361_adc.hdl", hdl_library="devices")
'hdl/devices/ad9361_adc.hdl'
>>> _get_asset_dir(noun="worker", name="matchstiq_z1_avr.hdl", hdl_platform="matchstiq_z1")
'hdl/platforms/matchstiq_z1/devices/matchstiq_z1_avr.hdl'
>>> _get_asset_dir(noun="applications")
'applications'
>>> _get_asset_dir(noun="application", name="rx_app")
'applications/rx_app'
>>> _get_asset_dir(noun="application", name="rx_app")
'applications/rx_app'
>>> _get_asset_dir(noun="hdl-assemblies")
'hdl/assemblies'
>>> _get_asset_dir(noun="hdl-assembly", name="fsk_modem")
'hdl/assemblies/fsk_modem'
>>> _get_asset_dir(noun="hdl-assembly", name="fsk_modem")
'hdl/assemblies/fsk_modem'
>>> _get_asset_dir(noun="hdl-primitive", name="zynq")
'hdl/primitives/zynq'
>>> _get_asset_dir(noun="hdl-primitive", name="zynq")
'hdl/primitives/zynq'
>>> _get_asset_dir(noun="hdl-library", name="devices")
'hdl/devices'
>>> _get_asset_dir(noun="hdl-platform", name="matchstiq_z1")
'hdl/platforms/matchstiq_z1'
>>> _get_asset_dir(noun="project", name="assets")
'assets'
>>> _get_asset_dir(noun="project")
'.'
>>> _get_asset_dir(library="components", hdl_library="devices", hdl_platform="matchstiq_z1")
Traceback (most recent call last):
...
OCPIException: ...
>>> _get_asset_dir(noun="INVALID")
Traceback (most recent call last):
...
OCPIException: ...
"""
#TODO if i do show project in a lower level dir it should use find project top to locate the
# directory of the project
if noun == "test" and not name.endswith((".test", ".test/")):
name += ".test"
# library-directives are mutually exclusive
if library is not None and hdl_library is not None:
#raise OCPIException("library, hdl_library, and hdl_platform options are " +
raise OCPIException("library and hdl_library are mutually exclusive.")
# asset types that live inside a library
library_assets = ["worker", "test"]
# For most nouns, there is only a single directory to choose from
if noun == "libraries" and library is None:
asset = "components"
elif noun in ["application", "applications"]:
asset = "applications"
elif noun in ["hdl-primitive", "hdl-primitives"]:
asset = "hdl/primitives"
elif noun == "hdl-platforms":
asset = "hdl/platforms"
elif noun in ["hdl-assembly", "hdl-assemblies"]:
asset = "hdl/assemblies"
elif noun == "hdl-platform" or hdl_platform is not None:
# hdl platform is specified in some way
# cannot have a platform noun AND directive
if noun == "hdl-platform" and name is not "." and hdl_platform is not None:
raise OCPIException(
"Could not choose between two HDL platform directories: '" +
name + "' and '" + hdl_platform + "'.")
# default hdl platform if needed
hdl_platform = name if hdl_platform is None else hdl_platform
# hdl_platform location directive or noun implies directories
# like hdl/platforms/<hdl-platform>
if noun in library_assets + ["library", "workers", "tests"]:
# can only specify library as 'devices' in a platform directory
if noun == "library" and name not in [".", "devices", "devices/"]:
raise OCPIException(
"Only valid library in an HDL platform is 'devices'")
# if the hdl_platform location directive is used and a library-asset is being
# located, drill down into the platform's devices library
asset = "hdl/platforms/" + hdl_platform + "/devices"
else:
asset = "hdl/platforms/" + hdl_platform
elif noun == "library" or library is not None:
# library is specified in some way
# cannot have a library noun AND directive
if noun == "library" and name is not "." and library is not None:
raise OCPIException(
"Could not choose between two library directories: '" + name +
"' and '" + library + "'.")
# default library if needed
library = name if library is None else library
if library == "components" or "/" in library:
# library is either the components directory or a full path - use it as-is
asset = library
else:
# library is just a name (not a path) - prepend components/ to it
asset = "components/" + library
elif noun == "hdl-library" or hdl_library is not None:
# hdl library is specified in some way
# cannot have a library noun AND directive
if noun == "hdl-library" and name is not "." and hdl_library is not None:
raise OCPIException(
"Could not choose between two hdl_library directories: '" +
name + "' and '" + hdl_library + "'.")
# default hdl library if needed
hdl_library = name if hdl_library is None else hdl_library
# hdl libraries live in the hdl/ subtree
asset = "hdl/" + hdl_library
elif noun in library_assets:
asset = "components/"
elif noun in ["project", "workers", "tests"]:
# If we have gotten this far and a "workers" or "tests" is discovered, the only
# remaining asset type that it can be is "project".
# When locating a project, it is either the current project (name is none), or the
# the directory is specified by name
asset = "." if name is None else name
elif noun is None:
# when no noun or location-directive gives direction regarding where to look for an
# asset, assume the top-level of the project
asset = "."
else:
raise OCPIException("Invalid noun provided: " + str(noun))
# If the asset-type/noun is a collection-type, then we already know its location.
# E.g. If the noun is 'library' (a collection-type) and the name is 'dsp_comps',
# then by now we have obtained asset='components/dsp_comps'. Just return that.
# Otherwise append 'name' to the determined collection-type asset.
# E.g. If library directives describe a collection-type/asset named 'components/dsp_comps',
# and the noun is 'worker' with name 'complex_mixer.hdl', then this will result in
# 'components/dsp_comps/complex_mixer.hdl
if noun in COLLECTION_DIRTYPES or name is None:
return asset
else:
return asset + "/" + name
def get_ocpidev_working_dir(origin_path=".",
noun="",
name=".",
library=None,
hdl_library=None,
hdl_platform=None):
"""
Given an origin path, noun, name and location/library directives,
determine the target assets's directory to work in.
Basically, the arguments of this function describe the current state/location in a project
as well as a description of where to look for an asset of a certain name and type.
This function completes that mapping of:
'current-state/location + asset-description' -> asset-directory,
and returns the working directory of interest.
First, this function converts the current state/location to location/library directives
like library, hdl_library, hdl_platform and collection_type. So basically, convert the
current-state + location-directives, to just location-directives. These location-directives
along with descriptors like noun and name comprise an asset-description.
At that point, the _get_asset_dir() function can be called which converts an asset-description
to asset-directory. This nested function call does not require any current-state/location
information, and will provide the location of the target asset relative to a project top.
Finally, the path to the project of interest is prepended to this asset-directory and
the resulting full-path is returned.
Args:
origin_path: path from which we are operating - this provides runtime context that may
translate to location directives:
(library/hdl_library/hdl_platform/collection_type)
noun: noun/asset-type of interest. Defaults to blank
name: name of the asset to locate. Basically, what is the name of the asset of type <noun>
library: the library in which the asset of interest lives
hdl_library: the HDL library in which the asset of interest lives (in the hdl/ subtree)
hdl_platform: the HDL platform in which the asset of interest lives.
might itself be the asset
Return:
asset_dir: the full path to asset described via this function's arguments
Notes:
name and noun describe the name and type of asset being described.
The library, platform arguments are location-directives or modifiers
that dictate where to the asset should be found.
"""
logging.debug("Getting ocpidev working dir from options:\n" +
str((origin_path, noun, name, library, hdl_library,
hdl_platform)))
# If the noun is project, append name to origin_path to ensure we operate from within a project
# from here forward
if noun == "project":
origin_path = origin_path + "/" + name
name = "."
#from _opencpi.util import OCPIException
if not is_path_in_project(origin_path):
raise OCPIException(
"Path \"" + os.path.realpath(origin_path) +
"\" is not in a project, so its settings " +
"as a subdirectory of a project could not be determined.")
# if this is a collection type, we care about the current directory's dirtype,
# otherwise we want the current asset's containing collection's dirtype
cur_dirtype = get_dirtype(origin_path)
if cur_dirtype is not None and cur_dirtype in COLLECTION_DIRTYPES:
# operating on a collection-type, so just proceed as usual
working_dir = origin_path
dirtype = cur_dirtype
else:
# not operating on a collection-type. It is most convenient here
# to determine the location/library-directives associated with the
# asset's containing collection
working_dir = origin_path + "/.."
dirtype = get_dirtype(working_dir)
# if no name was provided, set name to the origin_path's basename to preserve that
# information as we move to the parent collection
if (name is None
or name == ".") and get_dirtype(origin_path) is not None:
name = os.path.basename(os.path.realpath(origin_path))
# error checking
if (library is not None or hdl_library
is not None) and dirtype in ["library", "hdl-library"]:
raise OCPIException(
"[hdl-]library option cannot be provided when operating in a " +
"directory of [hdl-]library type.")
if (hdl_platform is not None) and dirtype in "hdl_platform":
raise OCPIException(
"hdl-platform option cannot be provided when operating in a " +
"directory of hdl-platform type.")
working_basename = os.path.basename(os.path.realpath(working_dir))
# if origin path is the 'hdl' subdir, and library is set, prepend 'hdl/'
if dirtype is None and working_basename == "hdl" and library is not None:
library = "hdl/" + library
elif dirtype == "library":
# Determine where this library lives to choose which locatin-directive
# to set
parent_dt = get_dirtype(working_dir + "/..")
parent_basename = os.path.basename(
os.path.realpath(working_dir + "/.."))
library = working_basename
if parent_dt in ["project", "libraries"]:
# if the parent directory is type project or libraries, then this is
# a component library (e.g. components or a sublibrary of components)
library = working_basename
elif parent_dt == "hdl-platform":
# parent directory is an HDL platform, so set that directive
hdl_platform = parent_basename
elif parent_dt is None and parent_basename == "hdl":
# parent directory has no type and is named 'hdl'. So, this is an hdl library
# e.g. devices, cards, adapters
library = "hdl/" + working_basename
elif dirtype == "hdl-platform":
hdl_platform = working_basename
logging.debug(
"Getting ocpidev working dir from options (auxiliary function):\n" +
str((noun, name, library, hdl_library, hdl_platform, dirtype)))
# If no noun was specified, and the dirtype is a collection, set noun to dirtype
# For example, if a command was run in hdl/assemblies, but no noun was specified,
# then just set the noun to hdl-assemblies
if noun == "":
noun = dirtype if dirtype in COLLECTION_DIRTYPES else ""
# Now that the current state/directory has been converted to location directives,
# call this helper function to convert or state-less asset-description to an
# asset-directory from a project top
asset_from_pj_top = _get_asset_dir(noun, name, library, hdl_library,
hdl_platform)
proj_path = get_path_to_project_top(origin_path)
# prepend the asset-directory with the path to the project top
asset_dir = os.path.realpath(proj_path + "/" + asset_from_pj_top)
# ensure existence and return
if os.path.exists(asset_dir):
return asset_dir
else:
raise OCPIException("Determined working directory of \"" + asset_dir +
"\" that does " + "not exist.")
def set_vars_from_make(mk_file, mk_arg="", verbose=None):
"""
Collect a dictionary of variables from a makefile
--------------------------------------------------
First arg is .mk file to use
Second arg is make arguments needed to invoke correct output
The output can be an assignment or a target
Third arg is a verbosity flag
Return a dictionary of variable names mapped to values from make
OCPI_LOG_LEVEL>=6 will print stderr from make for user to see
OCPI_LOG_LEVEL>=10 will pass OCPI_DEBUG_MAKE to make command and will
print both stdout and stderr for user to see
"""
with open(os.devnull, 'w') as fnull:
make_exists = subprocess.Popen(["which", "make"],\
stdout=subprocess.PIPE, stderr=fnull).communicate()[0]
if make_exists is None or make_exists == "":
if verbose != None and verbose != "":
logging.error("The '\"make\"' command is not available.")
return 1
# If log level >= 10 set OCPI_DEBUG_MAKE=1 (max debug level)
ocpi_log_level = int(os.environ.get('OCPI_LOG_LEVEL', 0))
if ocpi_log_level >= 10:
mk_dbg = "OCPI_DEBUG_MAKE=1"
else:
mk_dbg = ""
# If mk_file is a "Makefile" then we use the -C option on the directory containing
# the makefile else (is a .mk) use the -f option on the file
if mk_file.endswith("/Makefile"):
make_cmd = "make " + mk_dbg + " -n -r -s -C " + os.path.dirname(mk_file) + " " + mk_arg
else:
make_cmd = "make " + mk_dbg + " -n -r -s -f " + mk_file + " " + mk_arg
logging.debug("Calling make via:" + str(make_cmd.split()))
child = subprocess.Popen(make_cmd.split(), stderr=subprocess.PIPE, stdout=subprocess.PIPE,
universal_newlines=True)
mk_output, mk_err = child.communicate()
# Print out output from make if log level is high
logging.debug("STDOUT output from Make (set_vars_from_make):\n" + str(mk_output))
# Print out stderr from make if log level is medium/high or if make returned error
if child.returncode != 0 or ocpi_log_level >= 6:
if mk_err:
logging.error("STDERR output from Make (set_vars_from_make):\n" + str(mk_err))
if child.returncode != 0:
raise OCPIException("The following make command returned an error:\n" + make_cmd)
try:
grep_str = re.search(r'(^|\n)[a-zA-Z_][a-zA-Z_]*=.*',
str(mk_output.strip())).group()
except AttributeError:
logging.warning("No variables are set from \"" + mk_file + "\"")
return None
assignment_strs = [x.strip() for x in grep_str.split(';') if len(x.strip()) > 0]
make_vars = {}
for var_assignment in assignment_strs:
var_name, var_val = var_assignment.split('=')
# If the value is an empty string or just matching quotes, assign [] as the value
if var_val == "\"\"" or var_val == "\'\'" or var_val == "":
assignment_value = []
else:
assignment_value = var_val.strip('"').strip().split(' ')
make_vars[var_name] = assignment_value
return make_vars