def update_test_details(self,
errors=None,
coredumps=None,
scylla_conf=False):
if self.create_stats:
update_data = {}
self._stats['test_details'][
'time_completed'] = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M")
if self.monitors and self.monitors.nodes:
test_start_time = self._stats['test_details']['start_time']
update_data['results'] = self.get_prometheus_stats()
grafana_dataset = self.monitors.get_grafana_screenshot_and_snapshot(
test_start_time)
self._stats['test_details'][
'grafana_screenshots'] = grafana_dataset.get(
'screenshots', [])
self._stats['test_details'][
'grafana_snapshots'] = grafana_dataset.get(
'snapshots', [])
self._stats['test_details'][
'grafana_annotations'] = self.monitors.upload_annotations_to_s3(
)
self._stats['test_details'][
'prometheus_data'] = self.monitors.download_monitor_data()
if self.db_cluster:
self._stats['setup_details'][
'db_cluster_details'] = self.get_db_cluster_details()
if self.db_cluster and scylla_conf and 'scylla_args' not in self._stats[
'setup_details'].keys():
node = self.db_cluster.nodes[0]
res = node.remoter.run(
'grep ^SCYLLA_ARGS /etc/sysconfig/scylla-server',
verbose=True)
self._stats['setup_details']['scylla_args'] = res.stdout.strip(
)
res = node.remoter.run('cat /etc/scylla.d/io.conf',
verbose=True)
self._stats['setup_details']['io_conf'] = remove_comments(
res.stdout.strip())
res = node.remoter.run('cat /etc/scylla.d/cpuset.conf',
verbose=True)
self._stats['setup_details']['cpuset_conf'] = remove_comments(
res.stdout.strip())
self._stats['status'] = self.status
update_data.update({
'status': self._stats['status'],
'setup_details': self._stats['setup_details'],
'test_details': self._stats['test_details']
})
if errors:
update_data.update({'errors': errors})
if coredumps:
update_data.update({'coredumps': coredumps})
self.update(update_data)
def process_content(raw, content, content_counter):
code = utils.decode_line(raw)
try:
process_java_code(code, content, content_counter)
except Exception as e:
comments = utils.remove_comments(code).split()
content.extend(comments)
content_counter.update(comments)
def checkKeywords(self, submission):
message = ''
passed = True
code = utils.remove_comments(submission)
for word in self.keywords:
if not re.compile(r'\b({0})\b'.format(word)).search(code):
passed=False
message = "You did not use '%s'" % word
return {'passed':passed, 'message':message}
def test_remove_comments(self):
text_file = '#COMENTARIOS\nCherryPy==3.2.4\nDjango==1.4.13\nIPTCInfo==1.9.5-6\nIon==0.6.4.2\n#COMENTARIO2\nJinja2==2.7\nMarkupSafe==0.18\nMySQL-python==1.2.3\nPIL==1.1.7-1\nPillow==2.1.0\nRoutes==2.0\nSQLAlchemy==0.5.8\nSouth==0.7.3\n'
expected_text_file = 'CherryPy==3.2.4\nDjango==1.4.13\nIPTCInfo==1.9.5-6\nIon==0.6.4.2\nJinja2==2.7\nMarkupSafe==0.18\nMySQL-python==1.2.3\nPIL==1.1.7-1\nPillow==2.1.0\nRoutes==2.0\nSQLAlchemy==0.5.8\nSouth==0.7.3'
self.assertEqual(
remove_comments(text_file),
expected_text_file,
'test_remove_comments ok'
)
def read_ele(ele_file):
with open(ele_file) as FH:
lines = FH.readlines()
lines = remove_comments(lines)
(n_faces,vert_per_face,A) = map(int,lines[0].split())
assert 3 == vert_per_face
assert 0 == A # Attributes
assert (n_faces + 1 ) == len(lines)
faces = np.empty((n_faces,vert_per_face))
for i in xrange(1,len(lines)):
sp_line = map(float,lines[i].split())
assert(4 == len(sp_line))
faces[i-1,:] = np.array(sp_line[1:])
return faces
def read_node(node_file):
with open(node_file) as FH:
lines = FH.readlines()
lines = remove_comments(lines)
(n_vert,D,A,B) = map(int,lines[0].split())
assert 2 == D # Dimension
assert 0 == A # Attributes
assert 0 == B # Boundary markers
assert (n_vert + 1 ) == len(lines)
vertices = np.empty((n_vert,2))
for i in xrange(1,len(lines)):
(v_id,x,y) = map(float,lines[i].split())
vertices[i-1,0] = x
vertices[i-1,1] = y
return vertices
def get_list_of_dependencies(self, filename, debug=False):
"""get_list_of_dependencies
return a list of the files that this file depends on
Args:
filename: the name of the file to analyze
Return:
A list of files that specify the dependenies
Raises:
IOError
"""
deps = []
if debug:
print "input file: " + filename
#filename needs to be a verilog file
if (filename.partition(".")[2] != "v"):
if debug:
print "File is not a recognized verilog source"
return False
fbuf = ""
#the name is a verilog file, try and open is
try:
filein = open(filename)
fbuf = filein.read()
filein.close()
except IOError as err:
#if debug:
# print "the file is not a full path... searching RTL"
#didn't find with full path, search for it
try:
filepath = utils.find_rtl_file_location(filename, self.user_paths)
filein = open(filepath)
fbuf = filein.read()
filein.close()
except IOError as err_int:
ModuleFactoryError("Couldn't find file %s in the RTL directory" % filename)
#we have an open file!
if debug:
print "found file!"
#strip out everything we can't use
fbuf = utils.remove_comments(fbuf)
include_fbuf = fbuf
#search for `include
while (not len(include_fbuf.partition("`include")[2]) == 0):
ifile_name = include_fbuf.partition("`include")[2]
ifile_name = ifile_name.splitlines()[0]
ifile_name = ifile_name.strip()
ifile_name = ifile_name.strip("\"")
if debug:
print "found an include " + ifile_name + " ",
if (ifile_name not in self.verilog_dependency_list) and (ifile_name not in self.verilog_file_list):
self.verilog_dependency_list.append(ifile_name)
if debug:
print "adding " + ifile_name + " to the dependency list"
else:
if debug:
print "... already in have it"
include_fbuf = include_fbuf.partition("`include")[2]
#remove the ports list and the module name
fbuf = fbuf.partition(")")[2]
#modules have lines that start with a '.'
str_list = fbuf.splitlines()
module_token = ""
done = False
while (not done):
for i in range (0, len(str_list)):
line = str_list[i]
#remove white spaces
line = line.strip()
if (line.startswith(".") and line.endswith(",")):
#if debug:
# print "found a possible module... with token: " + line
module_token = line
break
#check if we reached the last line
if (i >= len(str_list) - 1):
done = True
if (not done):
#found a possible module
#partitoin the fbuf
#if debug:
# print "module token " + module_token
module_string = fbuf.partition(module_token)[0]
fbuf = fbuf.partition(module_token)[2]
fbuf = fbuf.partition(";")[2]
str_list = fbuf.splitlines()
#get rid of everything before the possible module
while (len(module_string.partition(";")[2]) > 0):
module_string = module_string.partition(";")[2]
module_string = module_string.partition("(")[0]
module_string = module_string.strip("#")
module_string = module_string.strip()
m_name = module_string.partition(" ")[0]
if debug:
print "module name: " + m_name
if (not deps.__contains__(m_name)):
if debug:
print "adding it to the deps list"
deps.append(module_string.partition(" ")[0])
#mlist = module_string.splitlines()
#work backwords
#look for the last line that has a '('
#for i in range (0, len(mlist)):
# mstr = mlist[i]
# print "item: " + mlist[i]
# #mstr = mlist[len(mlist) - 1 - i]
# #mstr = mstr.strip()
# if (mstr.__contains__(" ")):
# if debug:
# print "found: " + mstr.partition(" ")[0]
# deps.append(mstr.partition(" ")[0])
# break
return deps
def has_dependencies(self, filename, debug = False):
"""has_dependencies
returns true if the file specified has dependencies
Args:
filename: search for dependencies with this filename
Return:
True: The file has dependencies.
False: The file doesn't have dependencies
Raises:
IOError
"""
if debug:
print "input file: " + filename
#filename needs to be a verilog file
if (filename.partition(".")[2] != "v"):
if debug:
print "File is not a recognized verilog source"
return False
fbuf = ""
#the name is a verilog file, try and open is
try:
filein = open(filename)
fbuf = filein.read()
filein.close()
except IOError as err:
if debug:
print "the file is not a full path, searching RTL... ",
#didn't find with full path, search for it
try:
#print "self.user_paths: %s" % (self.user_paths)
filepath = utils.find_rtl_file_location(filename, self.user_paths)
filein = open(filepath)
fbuf = filein.read()
filein.close()
except ModuleNotFound as err:
fbuf = ""
except IOError as err_int:
if debug:
print "couldn't find file in the RTL directory"
ModuleFactoryError("Couldn't find file %s in the RTL directory" % filename)
#we have an open file!
if debug:
print "found file!"
#strip out everything we can't use
fbuf = utils.remove_comments(fbuf)
#modules have lines that start with a '.'
str_list = fbuf.splitlines()
for item in str_list:
item = item.strip()
if (item.startswith(".")):
if debug:
print "found a module!"
return True
return False
def get_module_buffer_tags(buf, bus = "", keywords = [], user_paths = [], project_tags = {}, debug = False):
raw_buf = buf
tags = {}
tags["keywords"] = {}
tags["ports"] = {}
tags["module"] = ""
tags["parameters"] = {}
tags["arbiter_masters"] = []
in_task = False
end_module = False
ports = [
"input",
"output",
"inout"
]
#XXX only working with verilog at this time, need to extend to VHDL
#print "filename: %s" % filename
#find all the metadata
for key in keywords:
index = buf.find (key)
if (index == -1):
if debug:
print "didn't find substring for " + key
continue
if debug:
print "found substring for " + key
substring = buf.__getslice__(index, len(buf)).splitlines()[0]
if debug:
print "substring: " + substring
if debug:
print "found " + key + " substring: " + substring
substring = substring.strip()
substring = substring.strip("//")
substring = substring.strip("/*")
tags["keywords"][key] = substring.partition(":")[2]
#remove all the comments from the code
buf = utils.remove_comments(buf)
#print "no comments: \n\n" + buf
for substring in buf.splitlines():
if (len(substring.partition("module")[1]) == 0):
continue
module_string = substring.partition("module")[2]
module_string = module_string.strip(" ")
module_string = module_string.strip("(")
module_string = module_string.strip("#")
index = module_string.find(" ")
if (index != -1):
tags["module"] = module_string.__getslice__(0, index)
else:
tags["module"] = module_string
if debug:
print "module name: " + module_string
print tags["module"]
break
#find all the ports
#find the index of all the processing block
substrings = buf.splitlines()
input_count = buf.count("input")
output_count = buf.count("output")
inout_count = buf.count("inout")
ldebug = debug
define_dict = preprocessor.generate_define_table(raw_buf, user_paths, ldebug)
if 'defines' in project_tags:
for d in project_tags["defines"]:
define_dict[d] = project_tags["defines"][d]
#find all the USER_PARAMETER declarations
user_parameters = []
substrings = raw_buf.splitlines()
for substring in substrings:
substring = substring.strip()
if "USER_PARAMETER" in substring:
name = substring.partition(":")[2].strip()
user_parameters.append(name)
param_dict = {}
#find all the parameters
substrings = buf.splitlines()
for substring in substrings:
substring = substring.strip()
if ("parameter" in substring):
if debug:
print "found parameter!"
substring = substring.partition("parameter")[2].strip()
parameter_name = substring.partition("=")[0].strip()
parameter_value = substring.partition("=")[2].strip()
parameter_value = parameter_value.partition(";")[0].strip()
parameter_value = parameter_value.strip(',')
if debug:
print "parameter name: " + parameter_name
print "parameter value: " + parameter_value
if parameter_name not in user_parameters:
tags["parameters"][parameter_name] = parameter_value
param_dict[parameter_name] = parameter_value
#find all the IO's
for io in ports:
end_module = False
in_task = False
tags["ports"][io] = {}
substrings = buf.splitlines()
for substring in substrings:
substring = substring.strip()
if substring.startswith("endmodule"):
end_module = True
continue
#Only count one module per buffer
if end_module:
continue
if substring.startswith("task"):
#Sub tasks and functions declare inputs and outputs, don't count these
in_task = True
continue
if substring.startswith("function"):
in_task = True
continue
if substring.startswith("endtask"):
in_task = False
continue
if substring.startswith("endfunction"):
in_task = False
continue
if in_task:
continue
#if line doesn't start with an input/output or inout
if (not substring.startswith(io)):
continue
#if the line does start with input/output or inout but is used in a name then bail
if (not substring.partition(io)[2][0].isspace()):
continue
#one style will declare the port names after the ports list
if (substring.endswith(";")):
substring = substring.rstrip(";")
#the other stile will include the entire port definition within the port declaration
if (substring.endswith(",")):
substring = substring.rstrip(",")
if debug:
print "substring: " + substring
substring = substring.partition(io)[2]
if (len(substring.partition("reg")[1]) != 0):
if (len(substring.partition("reg")[0]) > 0) and \
(substring.partition("reg")[0][-1].isspace()):
#print "Substring: %s" % substring
substring = substring.partition("reg")[2]
substring = substring.strip()
max_val = -1
min_val = -1
if (len(substring.partition("]")[2]) != 0):
#we have a range to work with?
length_string = substring.partition("]")[0] + "]"
substring = substring.partition("]")[2]
substring = substring.strip()
length_string = length_string.strip()
if "wire" in length_string:
length_string = length_string.partition("wire")[2].strip()
length_string = length_string.strip()
#if debug:
#print "length string: " + length_string
ldebug = debug
#print "module name: %s" % tags["module"]
#print "parameters: %s" % str(param_dict)
length_string = preprocessor.resolve_defines(length_string, define_dict, param_dict, debug=ldebug)
length_string = preprocessor.evaluate_range(length_string)
length_string = length_string.partition("]")[0]
length_string = length_string.strip("[")
if debug:
print "length string: " + length_string
max_val = string.atoi(length_string.partition(":")[0])
min_val = string.atoi(length_string.partition(":")[2])
tags["ports"][io][substring] = {}
if (max_val != -1):
tags["ports"][io][substring]["max_val"] = max_val
tags["ports"][io][substring]["min_val"] = min_val
tags["ports"][io][substring]["size"] = (max_val + 1) - min_val
else:
tags["ports"][io][substring]["size"] = 1
tags["arbiter_masters"] = arbiter.get_number_of_arbiter_hosts(tags)
if debug:
print "input count: " + str(input_count)
print "output count: " + str(output_count)
print "inout count: " + str(inout_count)
print "\n"
if debug:
print "module name: " + tags["module"]
for key in tags["keywords"].keys():
print "key: " + key + ":" + tags["keywords"][key]
for io in ports:
for item in tags["ports"][io].keys():
print io + ": " + item
for key in tags["ports"][io][item].keys():
value = tags["ports"][io][item][key]
if (isinstance( value, int)):
value = str(value)
print "\t" + key + ":" + value
return tags
def _resolve_dependency_for_module(self, module_path):
vpos = 0
buf = ""
try:
filein = open(module_path)
buf = str(filein.read())
filein.close()
except IOError as e:
raise ModuleError("File %s not found" % module_path)
buf = utils.remove_comments(buf)
include_buf = buf
while len(include_buf.partition("`include")[2]) > 0:
ifname = include_buf.partition("`include")[2]
ifname = ifname.splitlines()[0]
ifname = ifname.strip()
ifname = ifname.strip("\"")
#self.logger.debug("Found ifname: %s" % ifname)
module = None
try:
filename = utils.find_module_filename(ifname, self.user_paths)
module_tags = vutils.get_module_tags(filename, self.user_paths)
module = Module(module_tags, path = filename, user_paths = self.user_paths, instance_name = None, is_include_file = True, depth = self.depth + 1)
except ModuleError:
self.logger.debug("Didn't find verilog module with filename: %s" % ifname)
module_tags = {}
module_tags["module"] = ifname
module = Module(module_tags, path = None, user_paths = self.user_paths, instance_name = None, is_include_file = True, depth = self.depth + 1)
module.set_vpos(vpos)
vpos + 1
include_buf = include_buf.partition("`include")[2]
for n in module.get_module_graph().nodes():
#m = module.get_module_graph().node[n]
#self.graph.add_node(n)
#self.graph.node[n] = m
#if n not in self.graph.nodes():
m = module.get_module_graph().node[n]
self.graph.add_node(id(m))
self.graph.node[id(m)] = m
self.graph.add_edges_from(module.get_module_graph().edges())
self.graph.add_edge(id(self), id(module))
self.logger.debug("Looking for actual modules...")
module_dict = self.find_modules_within_buffer(buf.partition(")")[2])
for instance in module_dict:
module_type = module_dict[instance]
self.logger.info("module_type: %s" % module_type)
module = None
#print "Module Type: %s" % module_type
try:
filename = utils.find_module_filename(module_type, self.user_paths)
#print "Filename: %s" % filename
module_tags = vutils.get_module_tags(filename, user_paths = self.user_paths)
#print "got tags..."
module = Module(module_tags, path = filename, user_paths = self.user_paths, instance_name = instance, is_include_file = False, depth = self.depth + 1)
except ModuleError:
#self.logger.debug("Didn't find verilog module with filename :%s" % module_type)
module_tags = {}
module_tags["module"] = module_type
module = Module(module_tags, path = None, user_paths = self.user_paths, instance_name = instance, is_include_file = False, depth = self.depth + 1)
for n in module.get_module_graph().nodes():
m = module.get_module_graph().node[n]
self.graph.add_node(id(m))
self.graph.node[id(m)] = m
module.set_vpos(vpos)
vpos + 1
#self.graph.add_nodes_from(module.get_module_graph().nodes())
self.graph.add_edges_from(module.get_module_graph().edges())
self.graph.add_edge(id(self), id(module))
def get_list_of_dependencies(self, filename, debug=False):
"""get_list_of_dependencies
return a list of the files that this file depends on
Args:
filename: the name of the file to analyze
Return:
A list of files that specify the dependenies
Raises:
IOError
"""
deps = []
if debug:
print "input file: " + filename
#filename needs to be a verilog file
if (filename.partition(".")[2] != "v"):
if debug:
print "File is not a recognized verilog source"
return False
fbuf = ""
#the name is a verilog file, try and open is
try:
filein = open(filename)
fbuf = filein.read()
filein.close()
except IOError as err:
#if debug:
# print "the file is not a full path... searching RTL"
#didn't find with full path, search for it
try:
filepath = utils.find_rtl_file_location(filename, self.user_paths)
filein = open(filepath)
fbuf = filein.read()
filein.close()
except IOError as err_int:
ModuleFactoryError("Couldn't find file %s in the RTL directory" %
filename)
#we have an open file!
if debug:
print "found file!"
#strip out everything we can't use
fbuf = utils.remove_comments(fbuf)
include_fbuf = fbuf
#search for `include
while (not len(include_fbuf.partition("`include")[2]) == 0):
ifile_name = include_fbuf.partition("`include")[2]
ifile_name = ifile_name.splitlines()[0]
ifile_name = ifile_name.strip()
ifile_name = ifile_name.strip("\"")
if debug:
print "found an include " + ifile_name + " ",
if (not self.verilog_dependency_list.__contains__(ifile_name)
and not self.verilog_file_list.__contains__(ifile_name)):
self.verilog_dependency_list.append(ifile_name)
if debug:
print "adding " + ifile_name + " to the dependency list"
else:
if debug:
print "... already in have it"
include_fbuf = include_fbuf.partition("`include")[2]
#remove the ports list and the module name
fbuf = fbuf.partition(")")[2]
#modules have lines that start with a '.'
str_list = fbuf.splitlines()
module_token = ""
done = False
while (not done):
module_token = ""
parameter_found = False
parameter_flag = False
parameter_debt = None
for i in range(0, len(str_list)):
line = str_list[i]
#remove white spaces
line = line.strip()
if "#" in line:
line = line.partition("#")[2]
parameter_found = True
if parameter_found:
if parameter_debt == 0:
parameter_found = False
parameter_flag = True
while ("(" in line) or (")" in line):
if "(" in line:
line = line.partition("(")[2]
if parameter_debt is None:
parameter_debt = 1
else:
parameter_debt += 1
else:
line = line.partition("(")[2]
parameter_debt -= 1
if (line.startswith(".") and line.endswith(",")):
#if debug:
# print "found a possible module... with token: " + line
module_token = line
continue
if ";" in line and len(module_token) > 0:
break
#check if we reached the last line
if (i >= len(str_list) - 1):
done = True
if (not done):
module_string = fbuf.partition(module_token)[0]
fbuf = fbuf.partition(module_token)[2]
fbuf = fbuf.partition(";")[2]
str_list = fbuf.splitlines()
#get rid of everything before the possible module
while (len(module_string.partition(";")[2]) > 0):
module_string = module_string.partition(";")[2].strip()
#Now we have a string that contains the module_type and name
module_string = module_string.partition("(")[0].strip()
m_name = ""
if parameter_found:
m_name = module_string.partition("#")[0].strip()
else:
m_name = module_string.partition(" ")[0].strip()
if m_name not in deps:
if debug:
print "adding it to the deps list"
deps.append(m_name)
return deps
def has_dependencies(self, filename, debug=False):
"""has_dependencies
returns true if the file specified has dependencies
Args:
filename: search for dependencies with this filename
Return:
True: The file has dependencies.
False: The file doesn't have dependencies
Raises:
IOError
"""
if debug:
print "input file: " + filename
#filename needs to be a verilog file
if (filename.partition(".")[2] != "v"):
if debug:
print "File is not a recognized verilog source"
return False
fbuf = ""
#the name is a verilog file, try and open is
try:
filein = open(filename)
fbuf = filein.read()
filein.close()
except IOError as err:
if debug:
print "the file is not a full path, searching RTL... ",
#didn't find with full path, search for it
try:
#print "self.user_paths: %s" % (self.user_paths)
filepath = utils.find_rtl_file_location(filename, self.user_paths)
filein = open(filepath)
fbuf = filein.read()
filein.close()
except ModuleError as err:
fbuf = ""
except IOError as err_int:
if debug:
print "couldn't find file in the RTL directory"
ModuleFactoryError("Couldn't find file %s in the RTL directory" %
filename)
#we have an open file!
if debug:
print "found file!"
#strip out everything we can't use
fbuf = utils.remove_comments(fbuf)
#modules have lines that start with a '.'
str_list = fbuf.splitlines()
for item in str_list:
item = item.strip()
if (item.startswith(".")):
if debug:
print "found a module!"
return True
return False
def get_module_buffer_tags(buf,
bus="",
keywords=[],
user_paths=[],
project_tags={},
debug=False):
raw_buf = buf
tags = {}
tags["keywords"] = {}
tags["ports"] = {}
tags["module"] = ""
tags["parameters"] = {}
tags["arbiter_masters"] = []
in_task = False
end_module = False
ports = ["input", "output", "inout"]
#XXX only working with verilog at this time, need to extend to VHDL
#print "filename: %s" % filename
#find all the metadata
for key in keywords:
index = buf.find(key)
if (index == -1):
if debug:
print "didn't find substring for " + key
continue
if debug:
print "found substring for " + key
substring = buf.__getslice__(index, len(buf)).splitlines()[0]
if debug:
print "substring: " + substring
if debug:
print "found " + key + " substring: " + substring
substring = substring.strip()
substring = substring.strip("//")
substring = substring.strip("/*")
tags["keywords"][key] = substring.partition(":")[2]
#remove all the comments from the code
buf = utils.remove_comments(buf)
#print "no comments: \n\n" + buf
for substring in buf.splitlines():
if (len(substring.partition("module")[1]) == 0):
continue
module_string = substring.partition("module")[2]
module_string = module_string.strip(" ")
module_string = module_string.strip("(")
module_string = module_string.strip("#")
index = module_string.find(" ")
if (index != -1):
tags["module"] = module_string.__getslice__(0, index)
else:
tags["module"] = module_string
if debug:
print "module name: " + module_string
print tags["module"]
break
#find all the ports
#find the index of all the processing block
substrings = buf.splitlines()
input_count = buf.count("input")
output_count = buf.count("output")
inout_count = buf.count("inout")
ldebug = debug
define_dict = preprocessor.generate_define_table(raw_buf, user_paths,
ldebug)
if 'defines' in project_tags:
for d in project_tags["defines"]:
define_dict[d] = project_tags["defines"][d]
#find all the USER_PARAMETER declarations
user_parameters = []
substrings = raw_buf.splitlines()
for substring in substrings:
substring = substring.strip()
if "USER_PARAMETER" in substring:
name = substring.partition(":")[2].strip()
user_parameters.append(name)
param_dict = {}
#find all the parameters
substrings = buf.splitlines()
for substring in substrings:
substring = substring.strip()
if ("parameter" in substring):
if debug:
print "found parameter!"
substring = substring.partition("parameter")[2].strip()
parameter_name = substring.partition("=")[0].strip()
parameter_value = substring.partition("=")[2].strip()
parameter_value = parameter_value.partition(";")[0].strip()
parameter_value = parameter_value.strip(',')
if debug:
print "parameter name: " + parameter_name
print "parameter value: " + parameter_value
if parameter_name not in user_parameters:
tags["parameters"][parameter_name] = parameter_value
param_dict[parameter_name] = parameter_value
#find all the IO's
for io in ports:
end_module = False
in_task = False
tags["ports"][io] = {}
substrings = buf.splitlines()
for substring in substrings:
substring = substring.strip()
if substring.startswith("endmodule"):
end_module = True
continue
#Only count one module per buffer
if end_module:
continue
if substring.startswith("task"):
#Sub tasks and functions declare inputs and outputs, don't count these
in_task = True
continue
if substring.startswith("function"):
in_task = True
continue
if substring.startswith("endtask"):
in_task = False
continue
if substring.startswith("endfunction"):
in_task = False
continue
if in_task:
continue
#if line doesn't start with an input/output or inout
if (not substring.startswith(io)):
continue
#if the line does start with input/output or inout but is used in a name then bail
if (not substring.partition(io)[2][0].isspace()):
continue
#one style will declare the port names after the ports list
if (substring.endswith(";")):
substring = substring.rstrip(";")
#the other stile will include the entire port definition within the port declaration
if (substring.endswith(",")):
substring = substring.rstrip(",")
if debug:
print "substring: " + substring
substring = substring.partition(io)[2]
if (len(substring.partition("reg")[1]) != 0):
if (len(substring.partition("reg")[0]) > 0) and \
(substring.partition("reg")[0][-1].isspace()):
#print "Substring: %s" % substring
substring = substring.partition("reg")[2]
substring = substring.strip()
max_val = -1
min_val = -1
if (len(substring.partition("]")[2]) != 0):
#we have a range to work with?
length_string = substring.partition("]")[0] + "]"
substring = substring.partition("]")[2]
substring = substring.strip()
length_string = length_string.strip()
if "wire" in length_string:
length_string = length_string.partition("wire")[2].strip()
length_string = length_string.strip()
#if debug:
#print "length string: " + length_string
ldebug = debug
#print "module name: %s" % tags["module"]
#print "parameters: %s" % str(param_dict)
length_string = preprocessor.resolve_defines(length_string,
define_dict,
param_dict,
debug=ldebug)
length_string = preprocessor.evaluate_range(length_string)
length_string = length_string.partition("]")[0]
length_string = length_string.strip("[")
if debug:
print "length string: " + length_string
max_val = string.atoi(length_string.partition(":")[0])
min_val = string.atoi(length_string.partition(":")[2])
tags["ports"][io][substring] = {}
if (max_val != -1):
tags["ports"][io][substring]["max_val"] = max_val
tags["ports"][io][substring]["min_val"] = min_val
tags["ports"][io][substring]["size"] = (max_val + 1) - min_val
else:
tags["ports"][io][substring]["size"] = 1
tags["arbiter_masters"] = arbiter.get_number_of_arbiter_hosts(tags)
if debug:
print "input count: " + str(input_count)
print "output count: " + str(output_count)
print "inout count: " + str(inout_count)
print "\n"
if debug:
print "module name: " + tags["module"]
for key in tags["keywords"].keys():
print "key: " + key + ":" + tags["keywords"][key]
for io in ports:
for item in tags["ports"][io].keys():
print io + ": " + item
for key in tags["ports"][io][item].keys():
value = tags["ports"][io][item][key]
if (isinstance(value, int)):
value = str(value)
print "\t" + key + ":" + value
return tags
def generate_define_table(filestring="", user_paths = [], debug = False):
"""Reads in a module as a buffer and returns a dictionary of defines
Generates a table of defines that can be used to resolve values.
If all the defines cannot be evaluated directly by the
current module then this will search all the included modules
Args:
filestring: A buffer from the module's file
Returns:
A dictionary of defines
Raises:
PreProcessorError
"""
define_dict = {}
#from a file string find all the defines and generate an entry into a
#dictionary
filestring = utils.remove_comments(filestring)
str_list = filestring.splitlines()
for item in str_list:
if debug: print "Working on: %s" % item
item = item.strip()
#look for include files
if item.startswith("`include"):
if debug: print "found an include: " + item
#read int the include file, strip away the comments
#then append everything to the end
item = item.partition("`include")[2]
item = item.strip()
item = item.strip("\"")
inc_file = utils.find_rtl_file_location(item, user_paths)
if debug: print "include file location: " + inc_file
#try and open the include file
try:
ifile = open(inc_file)
fs = ifile.read()
ifile.close()
except:
if item != "project_defines.v":
raise PreProcessorError("Error while attempting to the include file: %s" % inc_file)
try:
if debug:
print "got the new file string"
include_defines = generate_define_table(fs, user_paths)
if debug:
print "after include_define"
print "length of include defines: " + str(len(include_defines.keys()))
for key in include_defines.keys():
#append the values found in the include back in the local dictionary
if debug: print "working on: " + key
if (not define_dict.has_key(key)):
define_dict[key] = include_defines[key]
if debug: print "added new items onto the list"
## except TypeError as terr:
## print "Type Error: " + str(terr)
except:
if item != "project_defines.v":
raise PreProcessorError("Error while processing: %s: %s" %(item, sys.exc_info()[0]))
#print "error while processing : ", item, ": ", sys.exc_info()[0]
continue
if item.startswith("`define"):
#if the string starts with `define split the name and value into the dictionary
## if debug:
## print "found a define: " + item
item = item.partition("`define")[2]
item = item.strip()
if (len(item.partition(" ")[2]) > 0):
name = item.partition(" ")[0].strip()
value = item.partition(" ")[2].strip()
if debug:
print "added " + name + "\n\tWith value: " + value
define_dict[name] = value
continue
if (len(item.partition("\t")[2]) > 0):
name = item.partition("\t")[0].strip()
value = item.partition("\t")[2].strip()
if debug:
print "added " + name + "\n\tWith value: " + value
define_dict[name] = value
continue
if debug:
print "found a define without a value: " + item
return define_dict
def generate_define_table(filestring="", user_paths=[], debug=False):
"""Reads in a module as a buffer and returns a dictionary of defines
Generates a table of defines that can be used to resolve values.
If all the defines cannot be evaluated directly by the
current module then this will search all the included modules
Args:
filestring: A buffer from the module's file
Returns:
A dictionary of defines
Raises:
PreProcessorError
"""
import utils
define_dict = {}
#from a file string find all the defines and generate an entry into a
#dictionary
filestring = utils.remove_comments(filestring)
str_list = filestring.splitlines()
for item in str_list:
if debug: print "Working on: %s" % item
item = item.strip()
#look for include files
if item.startswith("`include"):
if debug: print "found an include: " + item
#read int the include file, strip away the comments
#then append everything to the end
item = item.partition("`include")[2]
item = item.strip()
item = item.strip("\"")
inc_file = utils.find_rtl_file_location(item, user_paths)
if debug: print "include file location: " + inc_file
#try and open the include file
try:
ifile = open(inc_file)
fs = ifile.read()
ifile.close()
except:
if item != "project_defines.v":
raise PreProcessorError(
"Error while attempting to the include file: %s" %
inc_file)
try:
if debug:
print "got the new file string"
include_defines = generate_define_table(fs, user_paths)
if debug:
print "after include_define"
print "length of include defines: " + str(
len(include_defines.keys()))
for key in include_defines.keys():
#append the values found in the include back in the local dictionary
if debug: print "working on: " + key
if (not define_dict.has_key(key)):
define_dict[key] = include_defines[key]
if debug: print "added new items onto the list"
## except TypeError as terr:
## print "Type Error: " + str(terr)
except:
if item != "project_defines.v":
raise PreProcessorError("Error while processing: %s: %s" %
(item, sys.exc_info()[0]))
#print "error while processing : ", item, ": ", sys.exc_info()[0]
continue
if item.startswith("`define"):
#if the string starts with `define split the name and value into the dictionary
## if debug:
## print "found a define: " + item
item = item.partition("`define")[2]
item = item.strip()
if (len(item.partition(" ")[2]) > 0):
name = item.partition(" ")[0].strip()
value = item.partition(" ")[2].strip()
if debug:
print "added " + name + "\n\tWith value: " + value
define_dict[name] = value
continue
if (len(item.partition("\t")[2]) > 0):
name = item.partition("\t")[0].strip()
value = item.partition("\t")[2].strip()
if debug:
print "added " + name + "\n\tWith value: " + value
define_dict[name] = value
continue
if debug:
print "found a define without a value: " + item
return define_dict
def _resolve_dependency_for_module(self, module_path):
vpos = 0
buf = ""
try:
filein = open(module_path)
buf = str(filein.read())
filein.close()
except IOError as e:
raise ModuleError("File %s not found" % module_path)
buf = utils.remove_comments(buf)
include_buf = buf
while len(include_buf.partition("`include")[2]) > 0:
ifname = include_buf.partition("`include")[2]
ifname = ifname.splitlines()[0]
ifname = ifname.strip()
ifname = ifname.strip("\"")
#self.logger.debug("Found ifname: %s" % ifname)
module = None
try:
filename = utils.find_module_filename(ifname, self.user_paths)
module_tags = vutils.get_module_tags(filename, self.user_paths)
module = Module(module_tags,
path=filename,
user_paths=self.user_paths,
instance_name=None,
is_include_file=True,
depth=self.depth + 1)
except ModuleError:
self.logger.debug(
"Didn't find verilog module with filename: %s" % ifname)
module_tags = {}
module_tags["module"] = ifname
module = Module(module_tags,
path=None,
user_paths=self.user_paths,
instance_name=None,
is_include_file=True,
depth=self.depth + 1)
module.set_vpos(vpos)
vpos + 1
include_buf = include_buf.partition("`include")[2]
for n in module.get_module_graph().nodes():
#m = module.get_module_graph().node[n]
#self.graph.add_node(n)
#self.graph.node[n] = m
#if n not in self.graph.nodes():
m = module.get_module_graph().node[n]
self.graph.add_node(id(m))
self.graph.node[id(m)] = m
self.graph.add_edges_from(module.get_module_graph().edges())
self.graph.add_edge(id(self), id(module))
self.logger.debug("Looking for actual modules...")
module_dict = self.find_modules_within_buffer(buf.partition(")")[2])
for instance in module_dict:
module_type = module_dict[instance]
self.logger.info("module_type: %s" % module_type)
module = None
#print "Module Type: %s" % module_type
try:
filename = utils.find_module_filename(module_type,
self.user_paths)
#print "Filename: %s" % filename
module_tags = vutils.get_module_tags(
filename, user_paths=self.user_paths)
#print "got tags..."
module = Module(module_tags,
path=filename,
user_paths=self.user_paths,
instance_name=instance,
is_include_file=False,
depth=self.depth + 1)
except ModuleError:
#self.logger.debug("Didn't find verilog module with filename :%s" % module_type)
module_tags = {}
module_tags["module"] = module_type
module = Module(module_tags,
path=None,
user_paths=self.user_paths,
instance_name=instance,
is_include_file=False,
depth=self.depth + 1)
for n in module.get_module_graph().nodes():
m = module.get_module_graph().node[n]
self.graph.add_node(id(m))
self.graph.node[id(m)] = m
module.set_vpos(vpos)
vpos + 1
#self.graph.add_nodes_from(module.get_module_graph().nodes())
self.graph.add_edges_from(module.get_module_graph().edges())
self.graph.add_edge(id(self), id(module))