def popAssign(token, fileName, stack, flags, state, mallocDict):
"""
This function is responsible for popping an assign operator.
Encountering a malloc afore the assign will bring about a type comparison.
:param token: The token popped.
:param fileName: The file name.
:param stack: The stack being popped.
:param flags: The flags of the stack.
:param state: The state of the stack.
:param infoDict: The information dictionary.
"""
# we encountered '=', so we are now ready for the variable identifier
if "assignments" not in state or not (
state["assignments"][-1].line == token.line
and state["assignments"][-1].column == token.column):
return
flags["assign"] = token
if "malloc" in flags:
if "mallocType" not in flags:
vera.report(fileName, token.line, "Malloc lacks a pointer cast.")
else:
del flags["mallocType"]
pointerIdentifier = state[
"identifier"] if "vars" not in state else state["vars"][-1]
mallocDict.setdefault(pointerIdentifier.value, [])
mallocDict[pointerIdentifier.value].append(flags["malloc"])
del flags["malloc"]
def indentationAnalysis():
"""
For every file we will use the recursive function acceptPairs, which in this case will
return the top-level bracer couples list, and we'll check for zero indentation between those.
"""
global state, stateIndex, index, end, parens, commentLineDic, stateTokens, allLines, indentation, file
for file in vera.getSourceFileNames():
parens = vera.getTokens(file, 1, 0, -1, -1,
["leftbrace", "rightbrace"])
stateTokens = vera.getTokens(file, 1, 0, -1, -1,
["switch", "break", "case", "default"])
commentTokens = vera.getTokens(file, 1, 0, -1, -1,
["ccomment", "cppcomment"])
commentLineDic = {}
for comment in commentTokens:
commentLineDic[comment.line] = comment
allLines = vera.getAllLines(file)
state, stateIndex, index = None, 0, 0
end = len(parens)
indentation = 0
topLevelBracers = acceptPairs()
if (index != end):
vera.report(file, parens[index].line, "Excessive closing bracket?")
indentation = -1
if topLevelBracers:
checkFragmentedIndentation(1, len(allLines) - 1, topLevelBracers)
def isPointerMissingNullCheck(fileName, mallocDict):
"""
We make a pass on the malloc dictionary to check if the malloc count is larger than zero, meaning
not every malloc allocation we encountered, has a corresponding malloc null check.
:param fileName: File to check.
:param mallocDict: Dictionary with mallocs that need to conform.
"""
for k, v in mallocDict.iteritems():
for malloc in v:
vera.report(
fileName, malloc.line,
"Pointer's '" + k + "' malloc is missing null pointer check.")
def popIdentifier(identifier, fileName, stack, flags, state, infoDict,
assignOpPresent):
"""
This function is responsible for popping an identifier.
If the identifier wasn't assigned, an error is produced.
:param identifier: The identifier popped.
:param fileName: The file name.
:param stack: The stack being popped.
:param flags: The flags of the stack.
:param state: The state of the stack.
:param infoDict: The information dictionary.
:param assignOpPresent: Is an assign operator present in the state.
"""
constDict, varDict, typeDict = infoDict["constDict"], infoDict[
"varDict"], infoDict["typeDict"]
# if this identifier isn't in the constDict it must be a regular variable
# let's check for intialization
if [
flag for flag in ["function", "structDefinition", "dereference"]
if flag in state
] or "type" not in state or identifier.value in libFunctions:
return
assigned = assignOpPresent and state["assignments"] and isBefore(
identifier, state["assignments"][-1])
if "vars" in state:
if state["vars"][-1].value != identifier.value:
return
elif identifier.value != state["identifier"].value:
return
varDict[identifier.value] = identifier
type = state["type"][:]
if "const" in state:
type = [state["const"]] + type
constDict[identifier.value] = identifier
if "stars" in state and state["stars"]:
type += state["stars"][-1]
flags["stars"] = state["stars"][-1][:]
typeDict[identifier.value] = type
if state["type"][0].type == "struct":
identifierType = state["type"][0].value + " " + ''.join(
[t.value for t in type])
else:
identifierType = ''.join([t.value for t in type])
if not assigned:
vera.report(
fileName, identifier.line, "Uninitialized identifier '" +
identifier.value + "' of type " + identifierType + ".")
def checkUppercaseUnderscore(file, dic, errorPrefix):
"""
Checks if the given dictionary(String,Token) contains identifiers
that need to conform to UPPERCASE_WITH_UNDERSCORE.
:param file: File to check.
:param dic: Dictionary with identifiers that need to conform.
:param errorPrefix: Add this to the beginning of error.
"""
for k, v in dic.iteritems():
result = re.search(uppercasePattern, k)
if result == None:
vera.report(
file, v.line, errorPrefix + " " + k +
" name isn't UPPERCASE_WITH_UNDERSCORE.")
def popRecursiveAux(stack, token, tokenList, fileName):
"""
This is the auxiliary function of the pop recursive function.
It is used to recursively pop the tokens out of the stack. In case a recursive token is found,
it is called recursively.
:param stack: The stack that is examined.
:param token: The token that is examined.
:param tokenList: The token list being examined.
:param fileName: The file name.
:return: Returns the correct format so far of the tokens examined.
"""
spaceTokens = token.value in ["}", "]"]
correctFormat = (" " + token.value) if spaceTokens else token.value
startToken = stack.pop()
nextToken = startToken
tokenList.insert(0, token)
#print("START> value:" + nextToken.value + " line:" + str(nextToken.line) + " column:" + str(nextToken.column))
while nextToken.type != recursiveTokenNames[token.type]:
if nextToken.type in recursiveTokenNames:
#print("NEXT RECURSIVE> value:" + nextToken.value + " line:" + str(nextToken.line) + " column:" + str(nextToken.column))
innerCorrectFormat = popRecursiveAux(stack, nextToken, tokenList,
fileName)
correctFormat = innerCorrectFormat + correctFormat
else:
#print("NEXT> value:" + nextToken.value + " line:" + str(nextToken.line) + " column:" + str(nextToken.column))
tokenList.insert(0, nextToken)
correctFormat = nextToken.value + correctFormat
nextToken = stack.pop()
endToken = nextToken
tokenList.insert(0, endToken)
spaceTokens = spaceTokens and endToken is not startToken
correctFormat = (endToken.value + " " +
correctFormat) if spaceTokens else (endToken.value +
correctFormat)
literalEndColumn = startToken.column + len(startToken.value) - 1
if spaceTokens and (token.column - 2 < literalEndColumn
or tokenList[-2].column - 2 < endToken.column):
error = "Spacing for identifier/literal in operator "+endToken.value+token.value+ \
", supposed to be '" + correctFormat +"'."
vera.report(fileName, token.line, error)
#print(correctFormat)
#print("FINISH> value:" + endToken.value + " line:" + str(endToken.line) + " column:" + str(endToken.column))
return correctFormat
def checkLineRangeIndentation(startingLine, endingLine):
"""
This function runs through the lines and checks for tab amount at the
start of each line equal to indentation global variable.
:param startingLine: The line to start at.
:param endingLine: The line to end at.
"""
global allLines, indentation, file, stateTokens, stateIndex, state, commentLineDic
#print("checking line, starting:"+str(startingLine)+ " ending:" + str(endingLine)+" indentation:"+str(indentation))
i = startingLine + 1
while i < endingLine:
if i in commentLineDic:
newLines = len(re.findall("\n", commentLineDic[i].value))
i += newLines + 1
continue
line = allLines[i - 1]
result = re.search(emptyLinePattern, line)
if (result != None):
i += 1
continue
indentationOffset = 1
if stateIndex < len(stateTokens) and state != None:
if state == "switch":
stateToken = stateTokens[stateIndex]
#print(str(stateToken.line)+","+str(i)+": "+state + " " + str(stateIndex) + " " + stateToken.type)
if stateToken.line == i:
stateIndex += 1
if stateToken.type == "break":
state = "switch"
if stateToken.type == "break":
indentationOffset += 1
#print(stateToken.type + " " + str(indentation)+" "+str(indentationOffset)+"\n"+line)
overallIndentation = indentation + indentationOffset
result = re.search("^\t{" + str(overallIndentation) + "}(?! ).*$",
line)
if (result == None):
vera.report(
file, i, "Bad line indentation, expected to start with " +
str(overallIndentation) + " tab(s).")
i += 1
def isVarBeforeFirstFunction(funcs, vars, types, fileName):
"""
This function adheres to variables not appearing before the first function.
:param funcs: The functions dictionary.
:param vars: The vaiablles dictionary.
:param types: The types dictionnay.
:param fileName: The file that is being checked.
:return:
"""
globalVars = [
token for identifier, token in vars.items()
if types[identifier][0].column == 0
]
if not globalVars or not funcs:
return
firstFunc = min([f for f in funcs.values()],
key=operator.attrgetter('line'))
firstVar = min(globalVars, key=operator.attrgetter('line'))
if firstVar.line > firstFunc.line:
vera.report(
fileName, firstVar.line, "Global variable '" + firstVar.value +
"' appears after the first function " + firstFunc.value + ".")
def checkCamelCase(fileName, dic):
"""
Checks if the given dictionary(String,Token) contains identifiers that need to conforom
to lowerCamelCase, meaning they need to start with either a lowercase letter, or two or more
uppercase letters.
:param file: File to check.
:param dic: Dictionary with identifiers that need to conform.
"""
for k, v in dic.iteritems():
result = re.findall(camelCasePattern, k)
if result is None:
vera.report(
fileName, v.line, "Bad char(s) " + str(result) +
" used for identifier " + k + ".")
result = re.search("^([A-Z]{2,}|[a-z])", k)
if result is None:
vera.report(
fileName, v.line,
"Identifier name does not conform to camelCase ,"
" can't start with digit or one capital letter in identifier "
+ v.value + ".")
def sanitizerAnalysis(sanirizerDir, sanitizerFile):
"""
This function analyses memory errors produced by the address, leak and memory sanitizers.
Initially the function calls sanitize from sanitizer.py, recieving the list of errors regexed from the report.
Later two attempts are made incase the identifier is missing from the report, once to load current knowledge
of identifiers and in case the identifier doesn't appear in the current knowledge another run is made on the file
to populate all identifiers.
:param sanirizerDir: The directory from which to load the sanitizer reports.
:param sanitizerFile: A sanitizer report file, if "None" then all rports are scanned.
"""
runtimeErrorList = sanitize(sanirizerDir, sanitizerFile)
pointerLineDict = {}
fileNameOnly = {os.path.basename(k): k for k, v in fileInfoDict.items()}
for (file, line, error), (injectVars,
appearsInFiles) in runtimeErrorList.iteritems():
if file not in fileInfoDict:
if file not in fileNameOnly:
return
file = fileNameOnly[file]
line, identifiers = int(line), None
if injectVars is not None:
populateIdentifierLines(file, pointerLineDict)
if not (pointerLineDict.has_key(file)
and pointerLineDict[file].has_key(line)):
lookupRecurrentIdentifiers(file, pointerLineDict)
if pointerLineDict.has_key(file) and pointerLineDict[file].has_key(
line):
identifiers = pointerLineDict[file][line]
error = injectVars(identifiers.keys())
scenariosString = "Memory:" + "[" + ",".join(
[os.path.splitext(scen)[0]
for scen in sorted(appearsInFiles)]) + "] "
error = scenariosString + error + ":MemoryEnd:"
vera.report(file, line, error)
def popRecursive(token, fileName, stack, state, flags, infoDict,
assignOpPresent, flagsAssignOpPresent):
"""
This function recursively pops recursive tokens.
Whenever a '{' or '[' or '(' is met, it is popped recursively.
The correct format is that which has a space between each recursive token and the next token, unless
the recursive token is followed by the closing recursive token of the same kind.
That means that an example in which the recursive token is followed promptly with another token,
which is not the closure of the same kind, or in which the following token is promptly followed by a closure token,
will produce an error.
Furthermore only the '{' or '[' tokens and their closures will require spacing, as dictated by conventions.
In the case of a char type, the only kind of possible intialization is '{ 0 }'.
The information surmised regarding malloc will enable comparison of the types being assigned, in case the variable
type is already loaded in the stack, but notwithstanding the lack of a loaded type, a supplemental check is made
to infer the type from previously gathered data.
:param token: The recursive token being popped.
:param fileName: The file name.
:param stack: The stack being popped.
:param state: The state of the stack.
:param flags: The flags of the stack.
:param infoDict: The information dictionary.
:param assignOpPresent: Is an assign operator present in the stack's state.
:param flagsAssignOpPresent: Is an assign operator present in the stack's flags.
"""
defineDict, typeDict = infoDict["defineDict"], infoDict["typeDict"]
tokenList = []
correctFormat = popRecursiveAux(stack, token, tokenList, fileName)
#print(correctFormat)
if not (assignOpPresent and not flagsAssignOpPresent):
if "tokenList" in flags:
flags["tokenList"] = tokenList + flags["tokenList"]
flags["correctFormat"] = correctFormat + flags["correctFormat"]
else:
flags["tokenList"] = tokenList
flags["correctFormat"] = correctFormat
elif token.type == "rightbrace" and state["type"][0].value == "char":
if correctFormat != "{ 0 }":
vera.report(fileName, token.line,
"Char array needs to be initialized with '{ 0 }'")
if stack and stack[-1].type == "identifier":
identifier = stack[-1]
if token.type == "rightbracket" and "identifier" in state \
and identifier.value == state["identifier"].value:
identifierColumn, leftBracketToken = identifier.column + len(
identifier.value) - 1, tokenList[0]
if leftBracketToken.column - 1 != identifierColumn:
error = "Spacing for identifier before operator" + tokenList[0].value + token.value + \
" supposed to be '" + identifier.value + flags["correctFormat"] + "'."
vera.report(fileName, token.line, error)
tokenList = flags[
"tokenList"] if "tokenList" in flags else tokenList
if flagsAssignOpPresent:
for t in tokenList:
if t.type == "identifier" and not defineDict.has_key(
t.value):
vera.report(
fileName, t.line, "Non #define identifier '" +
t.value + "' used to initialize array.")
elif token.type == "rightparen" and "malloc" in flags:
identifier = state["identifier"] if "vars" not in state else state[
"vars"][-1]
if "type" in state:
type = state["type"][:]
if "stars" in state and state["stars"]:
type += state["stars"][-1]
elif identifier.value in typeDict:
type = typeDict[identifier.value][:]
identifierType = ''.join([t.value for t in type])
mallocType = ''.join([t.value for t in tokenList[1:-1]])
flags["mallocType"] = mallocType
if identifierType != mallocType:
vera.report(fileName, token.line,
"Malloc's cast is to incorrect type, pointer '"+identifier.value+"' initialized as '" + \
identifierType + "' but cast was to '" + mallocType + "'.")
def popStar(token, fileName, flags, state): #Justin Bieber
"""
The star '*' was encountered, let's check it isn't the cast* star and if it isn't,
then it's the pointer definition's * star so let's check it's next to the type
with a space from the variable.
:param fileName: The file name.
:param stack: The stack being popped.
:param flags: The flags of the stack.
:param state: The state of the stack.
"""
if "type" not in state or "identifier" not in state or "stars" not in state or not state[
"stars"] or not state["stars"][-1]:
return
type, star = state["type"] + flags["stars"], state["stars"][-1].pop()
starList = state["stars"][-1]
if token.column != star.column or token.line != star.line:
return
index, typeLength = len(starList), len(state["type"]) - 1
identifier = state["identifier"]
startTypeList = [tt.value for tt in type[:typeLength + index + 1]]
endTypeList = [tt.value for tt in type[typeLength + index + 1:]]
nextToken = starList[-1] if starList else state["type"][-1]
if "commas" in state:
comma = state["commas"][-1]
identifier = state["vars"][-1]
if not starList:
nextToken = comma
startTypeList = [tt.value for tt in state["type"]] + [
" ", comma.value
] + [tt.value for tt in starList]
endTypeList = [tt.value for tt in flags["stars"][index:]]
startType = ''.join(startTypeList)
endType = ''.join(endTypeList)
differenceToNextInType = star.column - nextToken.column - len(
nextToken.value)
differenceToVar = identifier.column - star.column - 1
starCloseToIdentifier = index != len(
flags["stars"]) - 1 or differenceToVar == 1
starCloseToNextInType = differenceToNextInType == 0
if not starCloseToIdentifier:
vera.report(
fileName, star.line,
"Pointer's '*' does not have a single whitespace before identifier, '"
+ startType + endType + " " + identifier.value + "' instead of '" +
startType + " " * differenceToNextInType + endType + " " *
(differenceToVar) + identifier.value + "'.")
if not starCloseToNextInType:
betweenTypeAndIdentifier = " " if starCloseToIdentifier else " " * differenceToVar
vera.report(
fileName, star.line,
"Pointer's '*' not next to the pointer's type '" + startType +
endType + " " + identifier.value + "' instead of '" + startType +
" " * (differenceToNextInType) + endType +
betweenTypeAndIdentifier + identifier.value + "'.")
def handleToken(t, fileName):
"""
This functions handle the current token.
It also checks for main function appearing before struct defintion and more than one semicolon per line.
:param t: The token handled.
:param fileName: The file name.
:return:
"""
global parenCount, braceCount, returnCount, lastReturn, controlParenCount, controlState, controlToken, \
functionary, flags, lastSemicolon
if t.type == "leftparen":
parenCount += 1
elif t.type == "rightparen":
parenCount -= 1
elif t.type == "leftbrace":
braceCount += 1
elif t.type == "rightbrace":
braceCount -= 1
if braceCount == 0:
flags.pop("struct", None)
returnCount, lastReturn = 0, None
elif t.type == "for" or t.type == "if":
controlParenCount = 0
controlState = CONTROL
controlToken = t
elif t.type == "do" or t.type == "else":
controlState = EXPECTED_BLOCK
elif t.type == "while" and prev != "rightbrace":
controlParenCount = 0
controlState = CONTROL
controlToken = t
elif t.type == "return":
returnCount += 1
if returnCount > 1:
vera.report(
fileName, t.line,
"Only one 'return' token should be used in a function. Found another in line:"
+ str(lastReturn.line))
lastReturn = t
elif t.type == "struct" and braceCount == 0:
if functionary.has_key("main"):
vera.report(fileName, t.line,
"Main function appears before struct defintion. struct is at line " + str(t.line) + \
" , main appeared at line " + str(functionary["main"].line) + ".")
flags["struct"] = t
elif t.type == "semicolon" and "forState" not in flags:
if lastSemicolon != None and t.line == lastSemicolon.line:
vera.report(
fileName, t.line,
"More than one semicolon cannot appear in the same line unless in an if statement."
)
lastSemicolon = t
def handleControl(t, fileName, infoDict):
"""
This function handles a control token.
It also checks for legal lhs = rhs operations, for loops having all three loop arguments
and control structures having full blocks.
:param t: The control token.
:param fileName: The file name.
:param infoDict: The information dictionary.
"""
global controlState, controlParenCount, controlToken, flags, prev, controlIdentifier
constDict, mallocDict, varDict, defineDict, typeDict = infoDict["constDict"], infoDict["mallocDict"], infoDict["varDict"], \
infoDict["defineDict"], infoDict["typeDict"]
if controlState == CONTROL:
if t.type == "leftparen":
controlParenCount += 1
elif t.type == "rightparen":
controlParenCount -= 1
if controlParenCount == 0:
controlState = EXPECTED_BLOCK
if controlToken.type == "for" and flags["forIdentifiers"] < 3:
vera.report(
fileName, controlToken.line,
"For loops must have all three loop arguments present."
)
controlIdentifier, controlToken, flags = None, None, {}
if controlToken != None:
if controlToken.type == "if":
if t.type == "equal" or t.type == "notequal":
flags["equal"] = t
elif t.type == "identifier":
if prev == "not" and t.value in mallocDict and mallocDict[
t.value]:
malloc = mallocDict[t.value][0]
if isBefore(malloc, t):
mallocDict[t.value].pop(0)
elif "equal" in flags and controlIdentifier != None:
rhs = None
if constDict.has_key(t.value):
rhs = constDict[t.value]
if rhs == None and defineDict.has_key(t.value):
rhs = defineDict[t.value]
if varDict.has_key(
controlIdentifier.value) and typeDict[
controlIdentifier.
value][0].type != "const" and rhs != None:
vera.report(
fileName, t.line, "Equality checking '" +
controlIdentifier.value + " == " + rhs.value +
"' with "
"left-hand side argument as variable and right-hand side as const or #define."
)
if t.value == "NULL" and controlIdentifier.value in mallocDict and mallocDict[
controlIdentifier.value]:
malloc = mallocDict[controlIdentifier.value][0]
if isBefore(malloc, controlIdentifier):
mallocDict[controlIdentifier.value].pop(0)
elif controlIdentifier.value == "NULL" and t.value in mallocDict and mallocDict[
t.value]:
malloc = mallocDict[t.value][0]
if isBefore(malloc, t):
mallocDict[t.value].pop(0)
controlIdentifier = None
del flags["equal"]
elif controlIdentifier is None:
controlIdentifier = t
elif controlToken.type == "for":
if "forState" not in flags:
flags["forState"] = EXPECTED_IDENTIFIER
flags["forParenthesis"] = controlParenCount + 1
flags["forIdentifiers"], flags["forSemicolons"] = 0, 0
if t.type == "identifier" and flags[
"forState"] == EXPECTED_IDENTIFIER:
flags["forState"] = EXPECTED_SEMICOLON
flags["forIdentifiers"] += 1
elif t.type == "semicolon":
controlIdentifier = None
if flags["forState"] == EXPECTED_IDENTIFIER:
vera.report(
fileName, t.line,
"For loops must have all three loop arguments present."
)
elif flags["forState"] == EXPECTED_SEMICOLON:
flags["forSemicolons"] += 1
flags["forState"] = EXPECTED_IDENTIFIER
elif controlState == EXPECTED_BLOCK and t.type in controlTokenTypes:
if prev == "else" and t.type == "if":
pass
elif t.type != "leftbrace" and t.type != "else":
vera.report(fileName, t.line,
"Full block {} expected in the control structure.")
controlState = BLOCK
prev = t.type if t.type in controlTokenTypes | {"identifier"} else prev
def acceptPairs():
"""
A recursive function that collects the current level's bracer couples, and for each one found
activates itself recursively, so each bracer couple will look for it's own lower-level bracer couples.
Each '{}' curly bracket will be checked sperately for indentation on the same level and the expected tab amount.
"""
global file, parens, index, end, indentation, allLines, stateTokens, stateIndex, state
thisLevelsParens = []
while index != end:
nextToken = parens[index]
tokenValue = nextToken.value
if tokenValue == "{":
leftParenLine, leftParenColumn = nextToken.line, nextToken.column
thisLevelsParens.append(index)
startingIndex = index
index += 1
if (stateIndex < len(stateTokens)):
if (stateTokens[stateIndex].line < leftParenLine - 1):
vera.report(file, leftParenLine, "Control structure of type:"+stateTokens[stateIndex].type+ \
" is missing matching parenthesis")
elif stateTokens[stateIndex].line == leftParenLine - 1:
state = stateTokens[stateIndex].type
stateIndex += 1
indentation += 1
lowerLevelsParens = acceptPairs()
checkFragmentedIndentation(parens[startingIndex].line,
parens[index].line, lowerLevelsParens)
if index == end:
vera.report(file, leftParenLine,
"Opening curly bracket is not closed.")
return
nextToken = parens[index]
thisLevelsParens.append(index)
index += 1
rightParenLine = nextToken.line
rightParenColumn = nextToken.column
if (leftParenLine != rightParenLine):
if (leftParenColumn != rightParenColumn):
# make an exception for line continuation
leftLine = allLines[leftParenLine - 1]
rightLine = allLines[rightParenLine - 1]
if (leftLine[-1] != "\\") and (rightLine[-1] != "\\"):
vera.report(
file, rightParenLine,
"Closing curly bracket not in the same line or column,"
" or brackets are not both indented with tabs.")
if leftParenColumn != indentation:
vera.report(
file, leftParenLine,
"Curly bracket tab indentation incorrect, found " +
str(leftParenColumn) + " characters but expected " +
str(indentation) + " tabs.")
else:
indentation -= 1
return thisLevelsParens
#print("returing parenthesis list:")
#print(thisLevelsParens)
return thisLevelsParens