def _processFile(self): self.lAllObjects = [] for sLine in self.filecontent: lTokens = tokens.create(sLine.replace('\t', ' ').rstrip()) lObjects = [] for sToken in lTokens: lObjects.append(parser.item(sToken)) blank.classify(lObjects) whitespace.classify(lTokens, lObjects) comment.classify(lTokens, lObjects) preprocessor.classify(lTokens, lObjects) pragma.classify(lTokens, lObjects, self.lOpenPragmas, self.lClosePragmas, self.dVars) self.lAllObjects.extend(lObjects) self.lAllObjects.append(parser.carriage_return()) try: self.lAllObjects[0].set_filename(self.filename) except IndexError: pass design_file.tokenize(self.lAllObjects) post_token_assignments(self.lAllObjects) set_token_hierarchy_value(self.lAllObjects) self.oTokenMap = process_tokens(self.lAllObjects)
def test_multiple_symbols_on_single_line_1(self): sLine = "variable We1, We2, We3, Wy : BIT := '1';" lTokens = [] lTokens.append('variable') lTokens.append(' ') lTokens.append('We1') lTokens.append(',') lTokens.append(' ') lTokens.append('We2') lTokens.append(',') lTokens.append(' ') lTokens.append('We3') lTokens.append(',') lTokens.append(' ') lTokens.append('Wy') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('BIT') lTokens.append(' ') lTokens.append(':=') lTokens.append(' ') lTokens.append("'1'") lTokens.append(';') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_colons_with_spaces_with_colon_at_beginning_of_line(self): sLine = ' : this : will : test : commas : ' lTokens = [] lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('this') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('will') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('test') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('commas') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_open_parenthesis_with_spaces_with_parenthesis_at_beginning_of_line( self): sLine = ' ( this ( will ( test ( commas ( ' lTokens = [] lTokens.append(' ') lTokens.append('(') lTokens.append(' ') lTokens.append('this') lTokens.append(' ') lTokens.append('(') lTokens.append(' ') lTokens.append('will') lTokens.append(' ') lTokens.append('(') lTokens.append(' ') lTokens.append('test') lTokens.append(' ') lTokens.append('(') lTokens.append(' ') lTokens.append('commas') lTokens.append(' ') lTokens.append('(') lTokens.append(' ') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_close_parenthesis_with_spaces_with_parenthesis_at_beginning_of_line( self): sLine = ' ) this ) will ) test ) commas ) ' lTokens = [] lTokens.append(' ') lTokens.append(')') lTokens.append(' ') lTokens.append('this') lTokens.append(' ') lTokens.append(')') lTokens.append(' ') lTokens.append('will') lTokens.append(' ') lTokens.append(')') lTokens.append(' ') lTokens.append('test') lTokens.append(' ') lTokens.append(')') lTokens.append(' ') lTokens.append('commas') lTokens.append(' ') lTokens.append(')') lTokens.append(' ') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_commas_with_spaces_with_comma_at_beginning_of_line(self): sLine = ' , this , will , test , commas , ' lTokens = [] lTokens.append(' ') lTokens.append(',') lTokens.append(' ') lTokens.append('this') lTokens.append(' ') lTokens.append(',') lTokens.append(' ') lTokens.append('will') lTokens.append(' ') lTokens.append(',') lTokens.append(' ') lTokens.append('test') lTokens.append(' ') lTokens.append(',') lTokens.append(' ') lTokens.append('commas') lTokens.append(' ') lTokens.append(',') lTokens.append(' ') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def update_line(self, sLine): ''' This method updates the line, lineLower and lineNoComment attributes. ''' self.line = sLine self.lineLower = sLine.lower() self.lineNoComment = utils.remove_comment(sLine) self.tokens, self.separators = tokens.create(sLine)
def test_single_quotes_in_comment(self): sLine = "--| 'a'" lTokens = [] lTokens.append("--| 'a'") lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_double_quotes_in_comment(self): sLine = '--| "yet another string"' lTokens = [] lTokens.append('--| "yet another string"') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_comment_only_line_without_spaces_before_the_comment(self): sLine = '-- This is a comment ' lTokens = [] lTokens.append('-- This is a comment ') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_quotes_in_comments(self): sLine = '--! some text "other text' lTokens = [] lTokens.append('--! some text "other text') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_double_dash_in_string_literal(self): sLine = ' x"--";' lTokens = [] lTokens.append(' ') lTokens.append('x') lTokens.append('"--"') lTokens.append(';') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_single_spaces(self): sLine = 'contents of line' lTokens = [] lTokens.append('contents') lTokens.append(' ') lTokens.append('of') lTokens.append(' ') lTokens.append('line') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_comment_at_end_of_line_without_spaces_around_dashes(self): sLine = 'contents of line--This is a comment' lTokens = [] lTokens.append('contents') lTokens.append(' ') lTokens.append('of') lTokens.append(' ') lTokens.append('line') lTokens.append('--This is a comment') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_single_quotes_around_spaces(self): sLine = "before = ' ' & after" lTokens = [] lTokens.append('before') lTokens.append(' ') lTokens.append('=') lTokens.append(' ') lTokens.append("' '") lTokens.append(' ') lTokens.append('&') lTokens.append(' ') lTokens.append('after') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_string_literals(self): sLine = '"this" "will" "test" "colons with multiple things in the quotes"' lTokens = [] lTokens.append('"this"') lTokens.append(' ') lTokens.append('"will"') lTokens.append(' ') lTokens.append('"test"') lTokens.append(' ') lTokens.append('"colons with multiple things in the quotes"') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_commas_without_spaces(self): sLine = 'this,will,test,commas,' lTokens = [] lTokens.append('this') lTokens.append(',') lTokens.append('will') lTokens.append(',') lTokens.append('test') lTokens.append(',') lTokens.append('commas') lTokens.append(',') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_multiple_character_tokens_near_end_of_line(self): sLine = ' a <= b **c' lTokens = [] lTokens.append(' ') lTokens.append('a') lTokens.append(' ') lTokens.append('<=') lTokens.append(' ') lTokens.append('b') lTokens.append(' ') lTokens.append('**') lTokens.append('c') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_plus_without_spaces_with_plus_at_beginning_of_line(self): sLine = '+this+will+test+colons+' lTokens = [] lTokens.append('+') lTokens.append('this') lTokens.append('+') lTokens.append('will') lTokens.append('+') lTokens.append('test') lTokens.append('+') lTokens.append('colons') lTokens.append('+') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_lessthan_equal_without_spaces_with_one_at_beginning_of_line(self): sLine = '<=this<=will<=test<=colons<=' lTokens = [] lTokens.append('<=') lTokens.append('this') lTokens.append('<=') lTokens.append('will') lTokens.append('<=') lTokens.append('test') lTokens.append('<=') lTokens.append('colons') lTokens.append('<=') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_not_equal_without_spaces_with_one_at_beginning_of_line(self): sLine = '/=this/=will/=test/=colons/=' lTokens = [] lTokens.append('/=') lTokens.append('this') lTokens.append('/=') lTokens.append('will') lTokens.append('/=') lTokens.append('test') lTokens.append('/=') lTokens.append('colons') lTokens.append('/=') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_star_star_without_spaces_with_one_at_beginning_of_line(self): sLine = '**this**will**test**colons**' lTokens = [] lTokens.append('**') lTokens.append('this') lTokens.append('**') lTokens.append('will') lTokens.append('**') lTokens.append('test') lTokens.append('**') lTokens.append('colons') lTokens.append('**') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_equal_greaterthan_without_spaces_with_one_at_beginning_of_line(self): sLine = '=>this=>will=>test=>colons=>' lTokens = [] lTokens.append('=>') lTokens.append('this') lTokens.append('=>') lTokens.append('will') lTokens.append('=>') lTokens.append('test') lTokens.append('=>') lTokens.append('colons') lTokens.append('=>') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_single_quote_without_spaces_with_quote_at_beginning_of_line(self): sLine = "'this'will'test'colons'" lTokens = [] lTokens.append("'") lTokens.append('this') lTokens.append("'") lTokens.append('will') lTokens.append("'") lTokens.append('test') lTokens.append("'") lTokens.append('colons') lTokens.append("'") lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_qualified_expression(self): sLine = " a => std_logic'('1')," lTokens = [] lTokens.append(' ') lTokens.append('a') lTokens.append(' ') lTokens.append('=>') lTokens.append(' ') lTokens.append('std_logic') lTokens.append("'") lTokens.append('(') lTokens.append("'1'") lTokens.append(')') lTokens.append(',') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def test_multiple_symbols_on_single_line_2(self): sLine = " DI_I : in std_logic_vector(N - 1 downto 0) := (others => 'X'); -- parallel data in" lTokens = [] lTokens.append(' ') lTokens.append('DI_I') lTokens.append(' ') lTokens.append(':') lTokens.append(' ') lTokens.append('in') lTokens.append(' ') lTokens.append('std_logic_vector') lTokens.append('(') lTokens.append('N') lTokens.append(' ') lTokens.append('-') lTokens.append(' ') lTokens.append('1') lTokens.append(' ') lTokens.append('downto') lTokens.append(' ') lTokens.append('0') lTokens.append(')') lTokens.append(' ') lTokens.append(':=') lTokens.append(' ') lTokens.append('(') lTokens.append('others') lTokens.append(' ') lTokens.append('=>') lTokens.append(' ') lTokens.append("'X'") lTokens.append(')') lTokens.append(';') lTokens.append(' ') lTokens.append('-- parallel data in') lActual = tokens.create(sLine) self.assertEqual(lTokens, lActual)
def __init__(self, line): self.line = line self.lineLower = line.lower() self.lineNoComment = utils.remove_comment(line) self.tokens, self.separators = tokens.create(self.line) self.indentLevel = None # Misc attributes self.isBlank = False # Comment attributes self.isComment = False self.hasComment = False self.hasInlineComment = False self.commentColumn = None # Library attributes self.isLibrary = False self.isLibraryUse = False # Entity attributes self.insideEntity = False self.isEntityDeclaration = False self.isEndEntityDeclaration = False # Port attributes self.insidePortMap = False self.isPortDeclaration = False self.isPortKeyword = False self.isEndPortMap = False # Generic attributes self.insideGenericMap = False self.isGenericDeclaration = False self.isGenericKeyword = False self.isEndGenericMap = False # Architecture attributes self.insideArchitecture = False self.isArchitectureBegin = False self.isArchitectureKeyword = False self.isEndArchitecture = False self.insideArchitectureDeclarativeRegion = False # Signal attributes self.isSignal = False self.insideSignal = False self.isEndSignal = False # Constant attributes self.insideConstant = False self.isConstant = False self.isConstantEnd = False self.isConstantArray = False # Variable attributes self.isVariable = False # Process attributes self.insideProcess = False self.isProcessBegin = False self.isProcessKeyword = False self.isProcessLabel = False self.isProcessEndLabel = False self.isProcessDeclarative = False self.isEndProcess = False self.insideSensitivityList = False self.isSensitivityListBegin = False self.isSensitivityListEnd = False self.isClockStatement = False self.insideClockProcess = False self.insideResetProcess = False self.isProcessIs = False # Concurrent attributes self.insideConcurrent = False self.isConcurrentBegin = False self.isEndConcurrent = False self.hasConcurrentLabel = False # When attributes self.insideWhen = False self.isWhenKeyword = False self.isWhenElseKeyword = False self.isWhenEnd = False # If attributes self.insideIf = False self.isElseKeyword = False self.isElseIfKeyword = False self.isEndIfKeyword = False self.isIfEnd = False self.isIfKeyword = False self.isThenKeyword = False self.isLastEndIf = False self.isFirstIf = False # Case attributes self.insideCaseStatement = False self.insideCase = False self.insideCaseWhen = False self.isCaseIsKeyword = False self.isCaseKeyword = False self.isCaseWhenEnd = False self.isCaseWhenKeyword = False self.isEndCaseKeyword = False self.isCaseNull = False self.hasCaseLabel = False self.hasEndCaseLabel = False # Sequential attributes self.insideSequential = False self.isSequentialEnd = False self.isSequential = False self.sequentialAlignmentColumn = None # Component attributes self.insideComponent = False self.isComponentDeclaration = False self.isComponentEnd = False # Instantiation attributes self.insideInstantiation = False self.isInstantiationDeclaration = False self.isDirectInstantiationDeclaration = False self.insideInstantiationPortMap = False self.isInstantiationPortKeyword = False self.isInstantiationPortEnd = False self.isInstantiationPortAssignment = False self.insideInstantiationGenericMap = False self.isInstantiationGenericKeyword = False self.isInstantiationGenericEnd = False self.isInstantiationGenericAssignment = False # Package attributes self.insidePackage = False self.isPackageKeyword = False self.isPackageEnd = False # Package Body attributes self.insidePackageBody = False self.isPackageBodyKeyword = False self.isPackageBodyEnd = False # Generate attributes self.insideGenerate = False self.isGenerateBegin = False self.isGenerateKeyword = False self.isGenerateEnd = False self.isGenerateLabel = False self.isGenerateEndLabel = False self.insideGenerateCase = False self.insideGenerateCaseWhen = False self.isGenerateCaseWhen = False # Function attributes self.insideFunction = False self.insideFunctionDeclarative = False self.isFunctionParameter = False self.isFunctionParameterEnd = False self.isFunctionBegin = False self.isFunctionKeyword = False self.isFunctionEnd = False self.isFunctionReturn = False self.hasFunctionReturnType = False self.isFunctionReturnKeyword = False self.hasFunctionIs = False # For Loop attributes self.insideForLoop = False self.isForLoopKeyword = False self.isForLoopEnd = False self.isForLoopLabel = False # While Loop attributes self.insideWhileLoop = False self.isWhileLoopKeyword = False self.isWhileLoopEnd = False # Type attributes self.isTypeKeyword = False self.isTypeEnd = False # Subtype attributes self.insideSubtype = False self.isSubtypeKeyword = False self.isSubtypeEnd = False # Enumerated Type attributes self.insideTypeEnumerated = False self.isTypeEnumeratedKeyword = False self.isTypeEnumeratedEnd = False # Type Array attributes self.insideTypeArray = False self.isTypeArrayKeyword = False self.isTypeArrayEnd = False # Type Record attributes self.insideTypeRecord = False self.isTypeRecordKeyword = False self.isTypeRecordEnd = False # Variable Assignment attributes self.insideVariableAssignment = False self.isVariableAssignmentEnd = False self.isVariableAssignment = False self.variableAssignmentAlignmentColumn = None # Assert attributes self.isAssertKeyword = False self.isAssertEnd = False self.insideAssert = False # With attributes self.isWithKeyword = False # Attribute attributes self.isAttributeKeyword = False self.isAttributeEnd = False self.insideAttribute = False # File attributes self.isFileKeyword = False self.isFileEnd = False self.insideFile = False # Procedure attributes self.insideProcedure = False self.insideProcedureDeclarative = False self.isProcedureParameter = False self.isProcedureParameterEnd = False self.isProcedureBegin = False self.isProcedureKeyword = False self.isProcedureEnd = False self.isProcedureReturn = False self.isProcedureIs = False # Block attributes self.insideBlock = False self.isBlockBegin = False self.isBlockKeyword = False self.isEndBlock = False # Wait attributes self.isWait = False # Code tags self.hasCodeTag = False self.codeTags = {} # After attributes self.hasAfterKeyword = False