class TestLexer(TestCase): def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-1.2 # Comment. DataLicense: CC0-1.0 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 5) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 5) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper( self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper( self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper( self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line
class TestLexer(object): def __init__(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-1.2 # Comment. DataLicense: CC0-1.0 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper( self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2) self.token_assert_helper( self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper( self.l.token(), 'DOC_COMMENT', 'DocumentComment', 5) self.token_assert_helper( self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 5) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper( self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper(self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper(self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper(self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def token_assert_helper(self, token, type, value, line): assert token.type == type assert token.value == value assert token.lineno == line
class TestLexer(TestCase): def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-2.1 # Comment. DataLicense: CC0-1.0 DocumentName: Sample_Document-V2.1 SPDXID: SPDXRef-DOCUMENT DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5) self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1', 5) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6) self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE', 'DocumentNamespace', 7) self.token_assert_helper( self.l.token(), 'LINE', 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301', 7) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 8) def test_external_document_references(self): data = ''' ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF', 'ExternalDocumentRef', 2) self.token_assert_helper(self.l.token(), 'DOC_REF_ID', 'DocumentRef-spdx-tool-2.1', 2) self.token_assert_helper( self.l.token(), 'DOC_URI', 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25' '04E0-4F89-41D3-9A0C-0305E82C3301', 2) self.token_assert_helper( self.l.token(), 'EXT_DOC_REF_CHKSUM', 'SHA1: ' 'd6a770ba38583ed4bb4525bd96e50461655d2759', 2) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper( self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper( self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper( self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def test_unknown_tag(self): data = ''' SomeUnknownTag: SomeUnknownValue ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line
class TestLexer(TestCase): maxDiff = None def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-2.1 # Comment. DataLicense: CC0-1.0 DocumentName: Sample_Document-V2.1 SPDXID: SPDXRef-DOCUMENT DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5) self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1', 5) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6) self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE', 'DocumentNamespace', 7) self.token_assert_helper(self.l.token(), 'LINE', 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301', 7) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 8) def test_external_document_references(self): data = ''' ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF', 'ExternalDocumentRef', 2) self.token_assert_helper(self.l.token(), 'DOC_REF_ID', 'DocumentRef-spdx-tool-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_URI', 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25' '04E0-4F89-41D3-9A0C-0305E82C3301', 2) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF_CHKSUM', 'SHA1: ' 'd6a770ba38583ed4bb4525bd96e50461655d2759', 2) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper(self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' SPDXID: SPDXRef-Package FilesAnalyzed: False PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ExternalRef: SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*: ExternalRefComment: <text>Some comment about the package.</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Package', 2) self.token_assert_helper(self.l.token(), 'PKG_FILES_ANALYZED', 'FilesAnalyzed', 3) self.token_assert_helper(self.l.token(), 'LINE', 'False', 3) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 4) self.token_assert_helper(self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 4) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 5) self.token_assert_helper(self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 5) self.token_assert_helper(self.l.token(), 'PKG_EXT_REF', 'ExternalRef', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:', 6) self.token_assert_helper(self.l.token(), 'PKG_EXT_REF_COMMENT', 'ExternalRefComment', 7) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some comment about the package.</text>', 7) def test_unknown_tag(self): data = ''' SomeUnknownTag: SomeUnknownValue ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2) def test_snippet(self): data = ''' SnippetSPDXID: SPDXRef-Snippet SnippetLicenseComments: <text>Some lic comment.</text> SnippetCopyrightText: <text>Some cr text.</text> SnippetComment: <text>Some snippet comment.</text> SnippetName: from linux kernel SnippetFromFileSPDXID: SPDXRef-DoapSource SnippetLicenseConcluded: Apache-2.0 LicenseInfoInSnippet: Apache-2.0 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'SNIPPET_SPDX_ID', 'SnippetSPDXID', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Snippet', 2) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_COMMENT', 'SnippetLicenseComments', 3) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some lic comment.</text>', 3) self.token_assert_helper(self.l.token(), 'SNIPPET_CR_TEXT', 'SnippetCopyrightText', 4) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some cr text.</text>', 4) self.token_assert_helper(self.l.token(), 'SNIPPET_COMMENT', 'SnippetComment', 5) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some snippet comment.</text>', 5) self.token_assert_helper(self.l.token(), 'SNIPPET_NAME', 'SnippetName', 6) self.token_assert_helper(self.l.token(), 'LINE', 'from linux kernel', 6) self.token_assert_helper(self.l.token(), 'SNIPPET_FILE_SPDXID', 'SnippetFromFileSPDXID', 7) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DoapSource', 7) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_CONC', 'SnippetLicenseConcluded', 8) self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 8) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_INFO', 'LicenseInfoInSnippet', 9) self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 9) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line