class TestLexer(TestCase):
def setUp(self):
self.l = Lexer()
self.l.build()
def test_document(self):
data = '''
SPDXVersion: SPDX-1.2
# Comment.
DataLicense: CC0-1.0
DocumentComment: <text>This is a sample spreadsheet</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion',
2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2)
self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense',
4)
self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4)
self.token_assert_helper(self.l.token(), 'DOC_COMMENT',
'DocumentComment', 5)
self.token_assert_helper(self.l.token(), 'TEXT',
'<text>This is a sample spreadsheet</text>',
5)
def test_creation_info(self):
data = '''
## Creation Information
Creator: Person: Gary O'Neall
Creator: Organization: Source Auditor Inc.
Creator: Tool: SourceAuditor-V1.2
Created: 2010-02-03T00:00:00Z
CreatorComment: <text>This is an example of an SPDX
spreadsheet format</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Gary O'Neall", 3)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4)
self.token_assert_helper(self.l.token(), 'ORG_VALUE',
'Organization: Source Auditor Inc.', 4)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5)
self.token_assert_helper(self.l.token(), 'TOOL_VALUE',
'Tool: SourceAuditor-V1.2', 5)
self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-03T00:00:00Z', 6)
def test_review_info(self):
data = '''
Reviewer: Person: Joe Reviewer
ReviewDate: 2010-02-10T00:00:00Z
ReviewComment: <text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Joe Reviewer", 2)
self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate',
3)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-10T00:00:00Z', 3)
self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT',
'ReviewComment', 4)
self.token_assert_helper(
self.l.token(), 'TEXT', '''<text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>''', 4)
def test_pacakage(self):
data = '''
PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'PKG_CHKSUM',
'PackageChecksum', 2)
self.token_assert_helper(
self.l.token(), 'CHKSUM',
'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2)
self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE',
'PackageVerificationCode', 3)
self.token_assert_helper(
self.l.token(), 'LINE',
'4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)',
3)
def token_assert_helper(self, token, ttype, value, line):
assert token.type == ttype
assert token.value == value
assert token.lineno == line
class TestLexer(object):
def __init__(self):
self.l = Lexer()
self.l.build()
def test_document(self):
data = '''
SPDXVersion: SPDX-1.2
# Comment.
DataLicense: CC0-1.0
DocumentComment: <text>This is a sample spreadsheet</text>
'''
self.l.input(data)
self.token_assert_helper(
self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2)
self.token_assert_helper(
self.l.token(), 'DOC_LICENSE', 'DataLicense', 4)
self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4)
self.token_assert_helper(
self.l.token(), 'DOC_COMMENT', 'DocumentComment', 5)
self.token_assert_helper(
self.l.token(), 'TEXT',
'<text>This is a sample spreadsheet</text>', 5)
def test_creation_info(self):
data = '''
## Creation Information
Creator: Person: Gary O'Neall
Creator: Organization: Source Auditor Inc.
Creator: Tool: SourceAuditor-V1.2
Created: 2010-02-03T00:00:00Z
CreatorComment: <text>This is an example of an SPDX
spreadsheet format</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Gary O'Neall", 3)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4)
self.token_assert_helper(self.l.token(), 'ORG_VALUE',
'Organization: Source Auditor Inc.', 4)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5)
self.token_assert_helper(self.l.token(), 'TOOL_VALUE',
'Tool: SourceAuditor-V1.2', 5)
self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-03T00:00:00Z', 6)
def test_review_info(self):
data = '''
Reviewer: Person: Joe Reviewer
ReviewDate: 2010-02-10T00:00:00Z
ReviewComment: <text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Joe Reviewer", 2)
self.token_assert_helper(
self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-10T00:00:00Z', 3)
self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT',
'ReviewComment', 4)
self.token_assert_helper(self.l.token(), 'TEXT',
'''<text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>''', 4)
def test_pacakage(self):
data = '''
PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'PKG_CHKSUM',
'PackageChecksum', 2)
self.token_assert_helper(self.l.token(), 'CHKSUM',
'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12',
2)
self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE',
'PackageVerificationCode', 3)
self.token_assert_helper(self.l.token(), 'LINE',
'4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)',
3)
def token_assert_helper(self, token, type, value, line):
assert token.type == type
assert token.value == value
assert token.lineno == line
class TestLexer(TestCase):
def setUp(self):
self.l = Lexer()
self.l.build()
def test_document(self):
data = '''
SPDXVersion: SPDX-2.1
# Comment.
DataLicense: CC0-1.0
DocumentName: Sample_Document-V2.1
SPDXID: SPDXRef-DOCUMENT
DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301
DocumentComment: <text>This is a sample spreadsheet</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion',
2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2)
self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense',
4)
self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4)
self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5)
self.token_assert_helper(self.l.token(), 'LINE',
'Sample_Document-V2.1', 5)
self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6)
self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE',
'DocumentNamespace', 7)
self.token_assert_helper(
self.l.token(), 'LINE',
'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301',
7)
self.token_assert_helper(self.l.token(), 'DOC_COMMENT',
'DocumentComment', 8)
self.token_assert_helper(self.l.token(), 'TEXT',
'<text>This is a sample spreadsheet</text>',
8)
def test_external_document_references(self):
data = '''
ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'EXT_DOC_REF',
'ExternalDocumentRef', 2)
self.token_assert_helper(self.l.token(), 'DOC_REF_ID',
'DocumentRef-spdx-tool-2.1', 2)
self.token_assert_helper(
self.l.token(), 'DOC_URI',
'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25'
'04E0-4F89-41D3-9A0C-0305E82C3301', 2)
self.token_assert_helper(
self.l.token(), 'EXT_DOC_REF_CHKSUM', 'SHA1: '
'd6a770ba38583ed4bb4525bd96e50461655d2759', 2)
def test_creation_info(self):
data = '''
## Creation Information
Creator: Person: Gary O'Neall
Creator: Organization: Source Auditor Inc.
Creator: Tool: SourceAuditor-V1.2
Created: 2010-02-03T00:00:00Z
CreatorComment: <text>This is an example of an SPDX
spreadsheet format</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Gary O'Neall", 3)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4)
self.token_assert_helper(self.l.token(), 'ORG_VALUE',
'Organization: Source Auditor Inc.', 4)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5)
self.token_assert_helper(self.l.token(), 'TOOL_VALUE',
'Tool: SourceAuditor-V1.2', 5)
self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-03T00:00:00Z', 6)
def test_review_info(self):
data = '''
Reviewer: Person: Joe Reviewer
ReviewDate: 2010-02-10T00:00:00Z
ReviewComment: <text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE',
"Person: Joe Reviewer", 2)
self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate',
3)
self.token_assert_helper(self.l.token(), 'DATE',
'2010-02-10T00:00:00Z', 3)
self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT',
'ReviewComment', 4)
self.token_assert_helper(
self.l.token(), 'TEXT', '''<text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>''', 4)
def test_pacakage(self):
data = '''
PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'PKG_CHKSUM',
'PackageChecksum', 2)
self.token_assert_helper(
self.l.token(), 'CHKSUM',
'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2)
self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE',
'PackageVerificationCode', 3)
self.token_assert_helper(
self.l.token(), 'LINE',
'4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)',
3)
def test_unknown_tag(self):
data = '''
SomeUnknownTag: SomeUnknownValue
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG',
'SomeUnknownTag', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2)
def token_assert_helper(self, token, ttype, value, line):
assert token.type == ttype
assert token.value == value
assert token.lineno == line
class TestLexer(TestCase):
maxDiff = None
def setUp(self):
self.l = Lexer()
self.l.build()
def test_document(self):
data = '''
SPDXVersion: SPDX-2.1
# Comment.
DataLicense: CC0-1.0
DocumentName: Sample_Document-V2.1
SPDXID: SPDXRef-DOCUMENT
DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301
DocumentComment: <text>This is a sample spreadsheet</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2)
self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4)
self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4)
self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5)
self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1',
5)
self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6)
self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE',
'DocumentNamespace', 7)
self.token_assert_helper(self.l.token(), 'LINE',
'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301',
7)
self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8)
self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 8)
def test_external_document_references(self):
data = '''
ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'EXT_DOC_REF',
'ExternalDocumentRef', 2)
self.token_assert_helper(self.l.token(), 'DOC_REF_ID',
'DocumentRef-spdx-tool-2.1', 2)
self.token_assert_helper(self.l.token(), 'DOC_URI',
'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25'
'04E0-4F89-41D3-9A0C-0305E82C3301', 2)
self.token_assert_helper(self.l.token(), 'EXT_DOC_REF_CHKSUM',
'SHA1: '
'd6a770ba38583ed4bb4525bd96e50461655d2759', 2)
def test_creation_info(self):
data = '''
## Creation Information
Creator: Person: Gary O'Neall
Creator: Organization: Source Auditor Inc.
Creator: Tool: SourceAuditor-V1.2
Created: 2010-02-03T00:00:00Z
CreatorComment: <text>This is an example of an SPDX
spreadsheet format</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4)
self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4)
self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5)
self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5)
self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6)
self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6)
def test_review_info(self):
data = '''
Reviewer: Person: Joe Reviewer
ReviewDate: 2010-02-10T00:00:00Z
ReviewComment: <text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2)
self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2)
self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3)
self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3)
self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4)
self.token_assert_helper(self.l.token(), 'TEXT', '''<text>This is just an example.
Some of the non-standard licenses look like they are actually
BSD 3 clause licenses</text>''', 4)
def test_pacakage(self):
data = '''
SPDXID: SPDXRef-Package
FilesAnalyzed: False
PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12
PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)
ExternalRef: SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:
ExternalRefComment: <text>Some comment about the package.</text>
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Package', 2)
self.token_assert_helper(self.l.token(), 'PKG_FILES_ANALYZED', 'FilesAnalyzed', 3)
self.token_assert_helper(self.l.token(), 'LINE', 'False', 3)
self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 4)
self.token_assert_helper(self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 4)
self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 5)
self.token_assert_helper(self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 5)
self.token_assert_helper(self.l.token(), 'PKG_EXT_REF', 'ExternalRef', 6)
self.token_assert_helper(self.l.token(), 'LINE', 'SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:', 6)
self.token_assert_helper(self.l.token(), 'PKG_EXT_REF_COMMENT', 'ExternalRefComment', 7)
self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some comment about the package.</text>', 7)
def test_unknown_tag(self):
data = '''
SomeUnknownTag: SomeUnknownValue
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2)
def test_snippet(self):
data = '''
SnippetSPDXID: SPDXRef-Snippet
SnippetLicenseComments: <text>Some lic comment.</text>
SnippetCopyrightText: <text>Some cr text.</text>
SnippetComment: <text>Some snippet comment.</text>
SnippetName: from linux kernel
SnippetFromFileSPDXID: SPDXRef-DoapSource
SnippetLicenseConcluded: Apache-2.0
LicenseInfoInSnippet: Apache-2.0
'''
self.l.input(data)
self.token_assert_helper(self.l.token(), 'SNIPPET_SPDX_ID', 'SnippetSPDXID', 2)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Snippet', 2)
self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_COMMENT', 'SnippetLicenseComments', 3)
self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some lic comment.</text>', 3)
self.token_assert_helper(self.l.token(), 'SNIPPET_CR_TEXT', 'SnippetCopyrightText', 4)
self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some cr text.</text>', 4)
self.token_assert_helper(self.l.token(), 'SNIPPET_COMMENT', 'SnippetComment', 5)
self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some snippet comment.</text>', 5)
self.token_assert_helper(self.l.token(), 'SNIPPET_NAME', 'SnippetName', 6)
self.token_assert_helper(self.l.token(), 'LINE', 'from linux kernel', 6)
self.token_assert_helper(self.l.token(), 'SNIPPET_FILE_SPDXID',
'SnippetFromFileSPDXID', 7)
self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DoapSource', 7)
self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_CONC',
'SnippetLicenseConcluded', 8)
self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 8)
self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_INFO',
'LicenseInfoInSnippet', 9)
self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 9)
def token_assert_helper(self, token, ttype, value, line):
assert token.type == ttype
assert token.value == value
assert token.lineno == line