def compileInstrumentedFile(self): source = self.getCodeFileNameIfExists() # Copy original command new_cmd = [COMPILER_NAME, '-include', FPCHECKER_RUNTIME ] + self.parameters # Replace file by instrumented file for i in range(len(new_cmd)): p = new_cmd[i] if p == source: new_cmd[i] = self.instrumentedFile break # Change output file if not self.outputFile: fileName, ext = os.path.splitext(source) newOutputFile = fileName + '.o' new_cmd = new_cmd + ['-o', newOutputFile] # Compile try: if verbose(): prGreen('Compiling: ' + ' '.join(new_cmd)) cmdOutput = subprocess.run(' '.join(new_cmd), shell=True, check=True) except Exception as e: if verbose(): prRed(e) logMessage(str(e)) message = 'Could not compile instrumented file' logMessage(message) raise CompileException(message) from e
def executePreprocessor(self): source = self.getCodeFileNameIfExists() # Copy the command parameters newParams = self.parameters.copy() outputFile = self.getOutputFileIfExists() if outputFile: self.preprocessedFile = outputFile + '.ii' for i in range(len(newParams)): p = self.parameters[i] if p == '-o' or p == '--output-file': newParams[i + 1] = self.preprocessedFile break else: self.preprocessedFile = source + '.ii' newParams.append('-o') newParams.append(self.preprocessedFile) new_cmd = [COMPILER_NAME, '-E'] + newParams try: if verbose(): prGreen(' '.join(new_cmd)) cmdOutput = subprocess.run(' '.join(new_cmd), shell=True, check=True) except Exception as e: message = 'Could not execute pre-processor' if verbose(): prRed(e) logMessage(str(e)) logMessage(message) raise RuntimeError(message) from e return True
def deprocess(self): tmpFd, tmpFname = tempfile.mkstemp(suffix='.txt', text=True) if verbose(): print('Temp file (deprocessing):', tmpFname) self.deprocessedFile = tmpFname dp = Deprocess(self.preFileName, tmpFname) if verbose(): print('Running de-processor...') dp.run() if verbose(): print('... de-preprocessor done.')
def match_assigment(self, tokensRange): startIndexes = self._find_indexes_with_assignmets(tokensRange) ret = [] for i in startIndexes: ## Match until ; m1 = self._match_anything_until_or_imbalanced_parenthesis(tokensRange[i:], ';') if m1: left = self._nextNonEmpty(tokensRange[i+1:]) if verbose(): print('PRE:', tokensRange[i+1+left]) if verbose(): print('POST:', tokensRange[i+m1-1]) ret.append((i+1+left, i+m1-1)) return ret
def executeOriginalCommand(self): try: cmd = [COMPILER_NAME] + self.parameters if verbose(): print('Executing original command:', cmd) subprocess.run(' '.join(cmd), shell=True, check=True) except subprocess.CalledProcessError as e: prRed(e)
def instrument(self): fileName, ext = os.path.splitext(self.sourceFileName) self.instrumentedFileName = fileName + '_inst' + ext with open(self.sourceFileName, 'r') as fd: with open(self.instrumentedFileName, 'w') as outFile: l = 0 for line in fd: l += 1 if l in self.transformedLines.keys(): if not self.is_omitted_line(self.sourceFileName, l): newLine = self.transformedLines[l] if verbose(): print(newLine[:-1]) outFile.write(newLine[:-1] + '\n') else: outFile.write(line[:-1] + '\n') else: if verbose(): print(line[:-1]) outFile.write(line[:-1] + '\n')
def instrumentSource(self): preFileName = self.preprocessedFile sourceFileName = self.getCodeFileNameIfExists() inst = Instrument(preFileName, sourceFileName) inst.deprocess() inst.findDeviceDeclarations() if verbose(): print(inst.deviceDclLines) inst.findAssigments() inst.produceInstrumentedLines() inst.instrument() self.instrumentedFile = inst.getInstrumentedFileName()
def tokenize(self): ## Create temp file and remove pre-processor lines (start with #) tmpFd, tmpFname = tempfile.mkstemp(suffix='.txt', text=True) if verbose(): print('Temp file (tokenizer):', tmpFname) with open(tmpFname, 'w') as f: with open(self.fileName, 'r') as src: for l in src: if l.startswith('#'): f.write('\n') else: f.write(l) ## Iterate on new file one character at a time with open(tmpFname, 'r') as f: while True: c = f.read(1) if not c: ## Match the final 2 tokens self.buff.append('\n') self.buff.append('\n') while True: token = self.match(self.buff) if not token or len(self.buff) == 2: break else: yield token if verbose(): print("\nEnd of file") break self.buff.append(c) token = self.match(self.buff) if token: yield token if token != None: continue os.close(tmpFd) if 'FPC_LEAVE_TEMP_FILES' not in os.environ: os.remove(tmpFname)
def produceInstrumentedLines(self): currentLine = 1 index = -1 while True: index += 1 if index >= len(self.allTokens): break token = self.allTokens[index] if str(token) == '\n': currentLine += 1 if currentLine in self.linesOfAssigments.keys(): self.transformedLines[currentLine] = '\n' if verbose(): print('[New Line (empty)]: ==>', self.transformedLines[currentLine]) continue if currentLine in self.linesOfAssigments.keys(): tokensConsumed, newLine = self.transformLine( index, currentLine) index += tokensConsumed - 1 if verbose(): print('[New Line]: ==>', newLine) self.transformedLines[currentLine] = newLine
def findAssigments(self): for l in self.deviceDclLines: startLine, endLine, startIndex, endIndex, f_type = l # unpack lines and indexes m = Match() tokenIndexes = m.match_assigment( self.allTokens[startIndex:endIndex]) for t in tokenIndexes: i_abs = startIndex + t[0] j_abs = startIndex + t[1] i_line = self.allTokens[i_abs].lineNumber() j_line = self.allTokens[j_abs].lineNumber() self.linesOfAssigments[i_line].append((i_abs, 'b')) self.linesOfAssigments[j_line].append((j_abs, 'e')) self.addMiddleLines(i_line, j_line) if verbose(): print('Lines with assigments:', self.linesOfAssigments) self.functionTypeMap[i_abs] = f_type
def match_device_function(self, buff): linesThatMatched = [] startIndexes = [] # index of __attribute__ tokens for i in range(len(buff)): if self._match_keyword(buff[i], '__attribute__'): startIndexes.append(i) ## Iterate starting from potential function definitions for i in startIndexes: if i+1+10 > len(buff): # we need at least 10 token to match continue #m1 = self._match_device_decl(buff[i:]) d, d_h, h_d, func_type = self._match_any_device_annotation(buff[i:]) if d or d_h or h_d: # Get the number of tokens fromn the annotation that matched if d: m1 = d elif d_h: m1 = d_h elif h_d: m1 = h_d else: return [] # we couldn't match any device function #print('m1', m1, 'func_type', func_type) m2 = self._match_anything_until(buff[i+m1:], '(') if m2: m3 = self._match_anything_until(buff[i+m1+m2:], ')') if m3: m4 = self._match_anything_until(buff[i+m1+m2+m3:], '{') if m4: m5 = self._match_anything_until_balanced_bracket(buff[i+m1+m2+m3+m4:]) if m5: startIndex = i endIndex = i+m1+m2+m3+m4+m5 - 1 # Important to subtract 1 (because indexes start with zero) startLine = buff[i].lineNumber() endLine = buff[endIndex].lineNumber() if not self._matched_block( (startLine, endLine) ): if verbose(): print('Not seen block:', (startLine, endLine), '\ncache:', self.code_range_cache) linesThatMatched.append((startLine, endLine, startIndex, endIndex, func_type)) return linesThatMatched
raise CompileException(message) from e if __name__ == '__main__': cmd = Command(sys.argv) if 'FPC_INSTRUMENT' not in os.environ: cmd.executeOriginalCommand() exit() # Link command if cmd.isLinkCommand(): cmd.executeOriginalCommand() else: # Compilation command try: cmd.executePreprocessor() cmd.instrumentSource() cmd.compileInstrumentedFile() logMessage('Instrumented: ' + cmd.instrumentedFile) except Exception as e: # Fall back to original command if verbose(): logMessage(str(e)) prRed(e) if not isinstance(e, EmptyFileException): logMessage('Failed: ' + ' '.join(sys.argv)) else: if verbose(): logMessage('Failed: ' + ' '.join(sys.argv)) cmd.executeOriginalCommand()
def printTokens(self, buff): for i in range(len(buff)): if verbose(): print('['+str(i)+']:', str(buff[i]))