def setUp(self): self.l = Lexer() self.l.build()
class TestLexer(TestCase): def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-2.1 # Comment. DataLicense: CC0-1.0 DocumentName: Sample_Document-V2.1 SPDXID: SPDXRef-DOCUMENT DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5) self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1', 5) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6) self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE', 'DocumentNamespace', 7) self.token_assert_helper( self.l.token(), 'LINE', 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301', 7) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 8) def test_external_document_references(self): data = ''' ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF', 'ExternalDocumentRef', 2) self.token_assert_helper(self.l.token(), 'DOC_REF_ID', 'DocumentRef-spdx-tool-2.1', 2) self.token_assert_helper( self.l.token(), 'DOC_URI', 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25' '04E0-4F89-41D3-9A0C-0305E82C3301', 2) self.token_assert_helper( self.l.token(), 'EXT_DOC_REF_CHKSUM', 'SHA1: ' 'd6a770ba38583ed4bb4525bd96e50461655d2759', 2) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper( self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper( self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper( self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def test_unknown_tag(self): data = ''' SomeUnknownTag: SomeUnknownValue ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line
class TestLexer(TestCase): def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-1.2 # Comment. DataLicense: CC0-1.0 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 5) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 5) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper( self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper( self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper( self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line
class Parser(object): def __init__(self, builder, logger): self.tokens = Lexer.tokens self.builder = builder self.logger = logger self.error = False self.license_list_parser = utils.LicenseListParser() self.license_list_parser.build(write_tables=0, debug=0) def p_start_1(self, p): 'start : start attrib ' pass def p_start_2(self, p): 'start : attrib ' pass def p_attrib(self, p): """attrib : spdx_version | data_lics | doc_comment | creator | created | creator_comment | locs_list_ver | reviewer | review_date | review_comment | package_name | package_version | pkg_down_location | pkg_home | pkg_summary | pkg_src_info | pkg_file_name | pkg_supplier | pkg_orig | pkg_chksum | pkg_verif | pkg_desc | pkg_lic_decl | pkg_lic_conc | pkg_lic_ff | pkg_lic_comment | pkg_cr_text | file_name | file_type | file_chksum | file_conc | file_lics_info | file_cr_text | file_lics_comment | file_notice | file_comment | file_contrib | file_dep | file_artifact | extr_lic_id | extr_lic_text | extr_lic_name | lic_xref | lic_comment | unknown_tag """ pass def more_than_one_error(self, tag, line): self.error = True msg = ERROR_MESSAGES['MORE_THAN_ONE'].format(tag, line) self.logger.log(msg) def order_error(self, first_tag, second_tag, line): """Reports an OrderError. Error message will state that first_tag came before second_tag. """ self.error = True msg = ERROR_MESSAGES['A_BEFORE_B'].format(first_tag, second_tag, line) self.logger.log(msg) def p_lic_xref_1(self, p): """lic_xref : LICS_CRS_REF LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.add_lic_xref(self.document, value) except OrderError: self.order_error('LicenseCrossReference', 'LicenseName', p.lineno(1)) def p_lic_xref_2(self, p): """lic_xref : LICS_CRS_REF error""" self.error = True msg = ERROR_MESSAGES['LICS_CRS_REF_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_lic_comment_1(self, p): """lic_comment : LICS_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_lic_comment(self.document, value) except OrderError: self.order_error('LicenseComment', 'LicenseID', p.lineno(1)) except CardinalityError: self.more_than_one_error('LicenseComment', p.lineno(1)) def p_lic_comment_2(self, p): """lic_comment : LICS_COMMENT error""" self.error = True msg = ERROR_MESSAGES['LICS_COMMENT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_extr_lic_name_1(self, p): """extr_lic_name : LICS_NAME extr_lic_name_value""" try: self.builder.set_lic_name(self.document, p[2]) except OrderError: self.order_error('LicenseName', 'LicenseID', p.lineno(1)) except CardinalityError: self.more_than_one_error('LicenseName', p.lineno(1)) def p_extr_lic_name_2(self, p): """extr_lic_name : LICS_NAME error""" self.error = True msg = ERROR_MESSAGES['LICS_NAME_VALE'].format(p.lineno(1)) self.logger.log(msg) def p_extr_lic_name_value_1(self, p): """extr_lic_name_value : LINE""" if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_extr_lic_name_value_2(self, p): """extr_lic_name_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_extr_lic_text_1(self, p): """extr_lic_text : LICS_TEXT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_lic_text(self.document, value) except OrderError: self.order_error('ExtractedText', 'LicenseID', p.lineno(1)) except CardinalityError: self.more_than_one_error('ExtractedText', p.lineno(1)) def p_extr_lic_text_2(self, p): """extr_lic_text : LICS_TEXT error""" self.error = True msg = ERROR_MESSAGES['LICS_TEXT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_extr_lic_id_1(self, p): """extr_lic_id : LICS_ID LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_lic_id(self.document, value) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['LICS_ID_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_extr_lic_id_2(self, p): """extr_lic_id : LICS_ID error""" self.error = True msg = ERROR_MESSAGES['LICS_ID_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_uknown_tag(self, p): """unknown_tag : UNKNOWN_TAG""" self.error = True msg = ERROR_MESSAGES['UNKNOWN_TAG'].format(p[1], p.lineno(1)) self.logger.log(msg) def p_file_artifact_1(self, p): """file_artifact : prj_name_art file_art_rest | prj_name_art """ pass def p_file_artificat_2(self, p): """file_artifact : prj_name_art error""" self.error = True msg = ERROR_MESSAGES['FILE_ART_OPT_ORDER'].format(p.lineno(2)) self.logger.log(msg) def p_file_art_rest(self, p): """file_art_rest : prj_home_art prj_uri_art | prj_uri_art prj_home_art | prj_home_art | prj_uri_art """ pass def p_prj_uri_art_1(self, p): """prj_uri_art : ART_PRJ_URI UN_KNOWN""" try: self.builder.set_file_atrificat_of_project(self.document, 'uri', utils.UnKnown()) except OrderError: self.order_error('ArtificatOfProjectURI', 'FileName', p.lineno(1)) def p_prj_uri_art_2(self, p): """prj_uri_art : ART_PRJ_URI LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_atrificat_of_project(self.document, 'uri', value) except OrderError: self.order_error('ArtificatOfProjectURI', 'FileName', p.lineno(1)) def p_prj_uri_art_3(self, p): """prj_uri_art : ART_PRJ_URI error""" self.error = True msg = ERROR_MESSAGES['ART_PRJ_URI_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_prj_home_art_1(self, p): """prj_home_art : ART_PRJ_HOME LINE""" try: self.builder.set_file_atrificat_of_project(self.document, 'home', p[2]) except OrderError: self.order_error('ArtificatOfProjectHomePage', 'FileName', p.lineno(1)) def p_prj_home_art_2(self, p): """prj_home_art : ART_PRJ_HOME UN_KNOWN""" try: self.builder.set_file_atrificat_of_project(self.document, 'home', utils.UnKnown()) except OrderError: self.order_error('ArtifactOfProjectName', 'FileName', p.lineno(1)) def p_prj_home_art_3(self, p): """prj_home_art : ART_PRJ_HOME error""" self.error = True msg = ERROR_MESSAGES['ART_PRJ_HOME_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_prj_name_art_1(self, p): """prj_name_art : ART_PRJ_NAME LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_atrificat_of_project(self.document, 'name', value) except OrderError: self.order_error('ArtifactOfProjectName', 'FileName', p.lineno(1)) def p_prj_name_art_2(self, p): """prj_name_art : ART_PRJ_NAME error""" self.error = True msg = ERROR_MESSAGES['ART_PRJ_NAME_VALUE'].format(p.lineno()) self.logger.log(msg) def p_file_dep_1(self, p): """file_dep : FILE_DEP LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.add_file_dep(self.document, value) except OrderError: self.order_error('FileDependency', 'FileName', p.lineno(1)) def p_file_dep_2(self, p): """file_dep : FILE_DEP error""" self.error = True msg = ERROR_MESSAGES['FILE_DEP_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_contrib_1(self, p): """file_contrib : FILE_CONTRIB LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.add_file_contribution(self.document, value) except OrderError: self.order_error('FileContributor', 'FileName', p.lineno(1)) def p_file_contrib_2(self, p): """file_contrib : FILE_CONTRIB error""" self.error = True msg = ERROR_MESSAGES['FILE_CONTRIB_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_notice_1(self, p): """file_notice : FILE_NOTICE TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_notice(self.document, value) except OrderError: self.order_error('FileNotice', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('FileNotice', p.lineno(1)) def p_file_notice_2(self, p): """file_notice : FILE_NOTICE error""" self.error = True msg = ERROR_MESSAGES['FILE_NOTICE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_cr_text_1(self, p): """file_cr_text : FILE_CR_TEXT file_cr_value""" try: self.builder.set_file_copyright(self.document, p[2]) except OrderError: self.order_error('FileCopyrightText', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('FileCopyrightText', p.lineno(1)) def p_file_cr_text_2(self, p): """file_cr_text : FILE_CR_TEXT error""" self.error = True msg = ERROR_MESSAGES['FILE_CR_TEXT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_cr_value_1(self, p): """file_cr_value : TEXT""" if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_file_cr_value_2(self, p): """file_cr_value : NONE""" p[0] = utils.SPDXNone() def p_file_cr_value_3(self, p): """file_cr_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_file_lics_comment_1(self, p): """file_lics_comment : FILE_LICS_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_license_comment(self.document, value) except OrderError: self.order_error('LicenseComments', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('LicenseComments', p.lineno(1)) def p_file_lics_comment_2(self, p): """file_lics_comment : FILE_LICS_COMMENT error""" self.error = True msg = ERROR_MESSAGES['FILE_LICS_COMMENT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_lics_info_1(self, p): """file_lics_info : FILE_LICS_INFO file_lic_info_value""" try: self.builder.set_file_license_in_file(self.document, p[2]) except OrderError: self.order_error('LicenseInfoInFile', 'FileName', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['FILE_LICS_INFO_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_lics_info_2(self, p): """file_lics_info : FILE_LICS_INFO error""" self.error = True msg = ERROR_MESSAGES['FILE_LICS_INFO_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_lic_info_value_1(self, p): """file_lic_info_value : NONE""" p[0] = utils.SPDXNone() def p_file_lic_info_value_2(self, p): """file_lic_info_value : NO_ASSERT""" p[0] = utils.NoAssert() # License Identifier def p_file_lic_info_value_3(self, p): """file_lic_info_value : LINE""" if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] p[0] = document.License.from_identifier(value) def p_conc_license_1(self, p): """conc_license : NO_ASSERT""" p[0] = utils.NoAssert() def p_conc_license_2(self, p): """conc_license : NONE""" p[0] = utils.SPDXNone() def p_conc_license_3(self, p): """conc_license : LINE""" if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] ref_re = re.compile('LicenseRef-.+', re.UNICODE) if (p[1] in config.LICENSE_MAP.keys()) or (ref_re.match(p[1]) is not None): p[0] = document.License.from_identifier(value) else: p[0] = self.license_list_parser.parse(value) def p_file_name_1(self, p): """file_name : FILE_NAME LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_name(self.document, value) except OrderError: self.order_error('FileName', 'PackageName', p.lineno(1)) def p_file_name_2(self, p): """file_name : FILE_NAME error""" self.error = True msg = ERROR_MESSAGES['FILE_NAME_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_comment_1(self, p): """file_comment : FILE_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_comment(self.document, value) except OrderError: self.order_error('FileComment', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('FileComment', p.lineno(1)) def p_file_comment_2(self, p): """file_comment : FILE_COMMENT error""" self.error = True msg = ERROR_MESSAGES['FILE_COMMENT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_type_1(self, p): """file_type : FILE_TYPE file_type_value""" try: self.builder.set_file_type(self.document, p[2]) except OrderError: self.order_error('FileType', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('FileType', p.lineno(1)) def p_file_type_2(self, p): """file_type : FILE_TYPE error""" self.error = True msg = ERROR_MESSAGES['FILE_TYPE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_chksum_1(self, p): """file_chksum : FILE_CHKSUM CHKSUM""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_file_chksum(self.document, value) except OrderError: self.order_error('FileChecksum', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('FileChecksum', p.lineno(1)) def p_file_chksum_2(self, p): """file_chksum : FILE_CHKSUM error""" self.error = True msg = ERROR_MESSAGES['FILE_CHKSUM_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_conc_1(self, p): """file_conc : FILE_LICS_CONC conc_license""" try: self.builder.set_concluded_license(self.document, p[2]) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['FILE_LICS_CONC_VALUE'].format(p.lineno(1)) self.logger.log(msg) except OrderError: self.order_error('LicenseConcluded', 'FileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('LicenseConcluded', p.lineno(1)) def p_file_conc_2(self, p): """file_conc : FILE_LICS_CONC error""" self.error = True msg = ERROR_MESSAGES['FILE_LICS_CONC_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_file_type_value(self, p): """file_type_value : OTHER | SOURCE | ARCHIVE | BINARY """ if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_pkg_desc_1(self, p): """pkg_desc : PKG_DESC TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_desc(self.document, value) except CardinalityError: self.more_than_one_error('PackageDescription', p.lineno(1)) except OrderError: self.order_error('PackageDescription', 'PackageFileName', p.lineno(1)) def p_pkg_desc_2(self, p): """pkg_desc : PKG_DESC error""" self.error = True msg = ERROR_MESSAGES['PKG_DESC_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_summary_1(self, p): """pkg_summary : PKG_SUM TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_summary(self.document, value) except OrderError: self.order_error('PackageSummary', 'PackageFileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageSummary', p.lineno(1)) def p_pkg_summary_2(self, p): """pkg_summary : PKG_SUM error""" self.error = True msg = ERROR_MESSAGES['PKG_SUM_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_cr_text_1(self, p): """pkg_cr_text : PKG_CPY_TEXT pkg_cr_text_value""" try: self.builder.set_pkg_cr_text(self.document, p[2]) except OrderError: self.order_error('PackageCopyrightText', 'PackageFileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageCopyrightText', p.lineno(1)) def p_pkg_cr_text_2(self, p): """pkg_cr_text : PKG_CPY_TEXT error""" self.error = True msg = ERROR_MESSAGES['PKG_CPY_TEXT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_cr_text_value_1(self, p): """pkg_cr_text_value : TEXT""" if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_pkg_cr_text_value_2(self, p): """pkg_cr_text_value : NONE""" p[0] = utils.SPDXNone() def p_pkg_cr_text_value_3(self, p): """pkg_cr_text_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_pkg_lic_comment_1(self, p): """pkg_lic_comment : PKG_LICS_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_license_comment(self.document, value) except OrderError: self.order_error('PackageLicenseComments', 'PackageFileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageLicenseComments', p.lineno(1)) def p_pkg_lic_comment_2(self, p): """pkg_lic_comment : PKG_LICS_COMMENT error""" self.error = True msg = ERROR_MESSAGES['PKG_LICS_COMMENT_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_decl_1(self, p): """pkg_lic_decl : PKG_LICS_DECL conc_license""" try: self.builder.set_pkg_license_declared(self.document, p[2]) except OrderError: self.order_error('PackageLicenseDeclared', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageLicenseDeclared', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_LICS_DECL_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_decl_2(self, p): """pkg_lic_decl : PKG_LICS_DECL error""" self.error = True msg = ERROR_MESSAGES['PKG_LICS_DECL_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_ff_1(self, p): """pkg_lic_ff : PKG_LICS_FFILE pkg_lic_ff_value""" try: self.builder.set_pkg_license_from_file(self.document, p[2]) except OrderError: self.order_error('PackageLicenseInfoFromFiles', 'PackageName', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_LIC_FFILE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_ff_value_1(self, p): """pkg_lic_ff_value : NONE""" p[0] = utils.SPDXNone() def p_pkg_lic_ff_value_2(self, p): """pkg_lic_ff_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_pkg_lic_ff_value_3(self, p): """pkg_lic_ff_value : LINE""" if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] p[0] = document.License.from_identifier(value) def p_pkg_lic_ff_2(self, p): """pkg_lic_ff : PKG_LICS_FFILE error""" self.error = True msg = ERROR_MESSAGES['PKG_LIC_FFILE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_conc_1(self, p): """pkg_lic_conc : PKG_LICS_CONC conc_license""" try: self.builder.set_pkg_licenses_concluded(self.document, p[2]) except CardinalityError: self.more_than_one_error('PackageLicenseConcluded', p.lineno(1)) except OrderError: self.order_error('PackageLicenseConcluded', 'PackageFileName', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_LICS_CONC_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_lic_conc_2(self, p): """pkg_lic_conc : PKG_LICS_CONC error""" self.error = True msg = ERROR_MESSAGES['PKG_LICS_CONC_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_src_info_1(self, p): """pkg_src_info : PKG_SRC_INFO TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_source_info(self.document, value) except CardinalityError: self.more_than_one_error('PackageSourceInfo', p.lineno(1)) except OrderError: self.order_error('PackageSourceInfo', 'PackageFileName', p.lineno(1)) def p_pkg_src_info_2(self, p): """pkg_src_info : PKG_SRC_INFO error""" self.error = True msg = ERROR_MESSAGES['PKG_SRC_INFO_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_chksum_1(self, p): """pkg_chksum : PKG_CHKSUM CHKSUM""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_chk_sum(self.document, value) except OrderError: self.order_error('PackageChecksum', 'PackageFileName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageChecksum', p.lineno(1)) def p_pkg_chksum_2(self, p): """pkg_chksum : PKG_CHKSUM error""" self.error = True msg = ERROR_MESSAGES['PKG_CHKSUM_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_verif_1(self, p): """pkg_verif : PKG_VERF_CODE LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_verif_code(self.document, value) except OrderError: self.order_error('PackageVerificationCode', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageVerificationCode', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_VERF_CODE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_verif_2(self, p): """pkg_verif : PKG_VERF_CODE error""" self.error = True msg = ERROR_MESSAGES['PKG_VERF_CODE_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_home_1(self, p): """pkg_home : PKG_HOME pkg_home_value""" try: self.builder.set_pkg_down_location(self.document, p[2]) except OrderError: self.order_error('PackageHomePage', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageHomePage', p.lineno(1)) def p_pkg_home_2(self, p): """pkg_home : PKG_HOME error""" self.error = True msg = ERROR_MESSAGES['PKG_HOME_VALUE'] self.logger.log(msg) def p_pkg_home_value_1(self, p): """pkg_home_value : LINE""" if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_pkg_home_value_2(self, p): """pkg_home_value : NONE""" p[0] = utils.SPDXNone() def p_pkg_home_value_3(self, p): """pkg_home_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_pkg_down_location_1(self, p): """pkg_down_location : PKG_DOWN pkg_down_value""" try: self.builder.set_pkg_down_location(self.document, p[2]) except OrderError: self.order_error('PackageDownloadLocation', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageDownloadLocation', p.lineno(1)) def p_pkg_down_location_2(self, p): """pkg_down_location : PKG_DOWN error""" self.error = True msg = ERROR_MESSAGES['PKG_DOWN_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_down_value_1(self, p): """pkg_down_value : LINE """ if six.PY2: p[0] = p[1].decode(encoding='utf-8') else: p[0] = p[1] def p_pkg_down_value_2(self, p): """pkg_down_value : NONE""" p[0] = utils.SPDXNone() def p_pkg_down_value_3(self, p): """pkg_down_value : NO_ASSERT""" p[0] = utils.NoAssert() def p_pkg_orig_1(self, p): """pkg_orig : PKG_ORIG pkg_supplier_values""" try: self.builder.set_pkg_originator(self.document, p[2]) except OrderError: self.order_error('PackageOriginator', 'PackageName', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_ORIG_VALUE'].format(p.lineno(1)) self.logger.log(msg) except CardinalityError: self.more_than_one_error('PackageOriginator', p.lineno(1)) def p_pkg_orig_2(self, p): """pkg_orig : PKG_ORIG error""" self.error = True msg = ERROR_MESSAGES['PKG_ORIG_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_supplier_1(self, p): """pkg_supplier : PKG_SUPPL pkg_supplier_values""" try: self.builder.set_pkg_supplier(self.document, p[2]) except OrderError: self.order_error('PackageSupplier', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageSupplier', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['PKG_SUPPL_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_supplier_2(self, p): """pkg_supplier : PKG_SUPPL error""" self.error = True msg = ERROR_MESSAGES['PKG_SUPPL_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_pkg_supplier_values_1(self, p): """pkg_supplier_values : NO_ASSERT""" p[0] = utils.NoAssert() def p_pkg_supplier_values_2(self, p): """pkg_supplier_values : entity""" p[0] = p[1] def p_pkg_file_name(self, p): """pkg_file_name : PKG_FILE_NAME LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_file_name(self.document, value) except OrderError: self.order_error('PackageFileName', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageFileName', p.lineno(1)) def p_pkg_file_name_1(self, p): """pkg_file_name : PKG_FILE_NAME error""" self.error = True msg = ERROR_MESSAGES['PKG_FILE_NAME_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_package_version_1(self, p): """package_version : PKG_VERSION LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_pkg_vers(self.document, value) except OrderError: self.order_error('PackageVersion', 'PackageName', p.lineno(1)) except CardinalityError: self.more_than_one_error('PackageVersion', p.lineno(1)) def p_package_version_2(self, p): """package_version : PKG_VERSION error""" self.error = True msg = ERROR_MESSAGES['PKG_VERSION_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_package_name(self, p): """package_name : PKG_NAME LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.create_package(self.document, value) except CardinalityError: self.more_than_one_error('PackageName', p.lineno(1)) def p_package_name_1(self, p): """package_name : PKG_NAME error""" self.error = True msg = ERROR_MESSAGES['PACKAGE_NAME_VALUE'].format(p.lineno(1)) self.logger.log(msg) def p_reviewer_1(self, p): """reviewer : REVIEWER entity""" self.builder.add_reviewer(self.document, p[2]) def p_reviewer_2(self, p): """reviewer : REVIEWER error""" self.error = True msg = ERROR_MESSAGES['REVIEWER_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_review_date_1(self, p): """review_date : REVIEW_DATE DATE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.add_review_date(self.document, value) except CardinalityError: self.more_than_one_error('ReviewDate', p.lineno(1)) except OrderError: self.order_error('ReviewDate', 'Reviewer', p.lineno(1)) def p_review_date_2(self, p): """review_date : REVIEW_DATE error""" self.error = True msg = ERROR_MESSAGES['REVIEW_DATE_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_review_comment_1(self, p): """review_comment : REVIEW_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.add_review_comment(self.document, value) except CardinalityError: self.more_than_one_error('ReviewComment', p.lineno(1)) except OrderError: self.order_error('ReviewComment', 'Reviewer', p.lineno(1)) def p_review_comment_2(self, p): """review_comment : REVIEW_COMMENT error""" self.error = True msg = ERROR_MESSAGES['REVIEW_COMMENT_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_lics_list_ver_1(self, p): """locs_list_ver : LIC_LIST_VER LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_lics_list_ver(self.document, value) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['LIC_LIST_VER_VALUE'].format( p[2], p.lineno(2)) self.logger.log(msg) except CardinalityError: self.more_than_one_error('LicenseListVersion', p.lineno(1)) def p_lics_list_ver_2(self, p): """locs_list_ver : LIC_LIST_VER error""" self.error = True msg = ERROR_MESSAGES['LIC_LIST_VER_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_doc_comment_1(self, p): """doc_comment : DOC_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_doc_comment(self.document, value) except CardinalityError: self.more_than_one_error('DocumentComment', p.lineno(1)) def p_doc_comment_2(self, p): """doc_comment : DOC_COMMENT error""" self.error = True msg = ERROR_MESSAGES['DOC_COMMENT_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_data_license_1(self, p): """data_lics : DOC_LICENSE LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_doc_data_lics(self.document, value) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['DOC_LICENSE_VALUE'].format(p[2], p.lineno(2)) self.logger.log(msg) except CardinalityError: self.more_than_one_error('DataLicense', p.lineno(1)) def p_data_license_2(self, p): """data_lics : DOC_LICENSE error""" self.error = True msg = ERROR_MESSAGES['DOC_LICENSE_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_spdx_version_1(self, p): """spdx_version : DOC_VERSION LINE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_doc_version(self.document, value) except CardinalityError: self.more_than_one_error('SPDXVersion', p.lineno(1)) except SPDXValueError: self.error = True msg = ERROR_MESSAGES['DOC_VERSION_VALUE'].format(p[2], p.lineno(1)) self.logger.log(msg) except IncompatibleVersionError: self.error = True self.logger.log( 'SPDXVersion must be SPDX-1.2 found {0}.'.format(value)) def p_spdx_version_2(self, p): """spdx_version : DOC_VERSION error""" self.error = True msg = ERROR_MESSAGES['DOC_VERSION_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_creator_comment_1(self, p): """creator_comment : CREATOR_COMMENT TEXT""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_creation_comment(self.document, value) except CardinalityError: self.more_than_one_error('CreatorComment', p.lineno(1)) def p_creator_comment_2(self, p): """creator_comment : CREATOR_COMMENT error""" self.error = True msg = ERROR_MESSAGES['CREATOR_COMMENT_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_creator_1(self, p): """creator : CREATOR entity""" self.builder.add_creator(self.document, p[2]) def p_creator_2(self, p): """creator : CREATOR error""" self.error = True msg = ERROR_MESSAGES['CREATOR_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_created_1(self, p): """created : CREATED DATE""" try: if six.PY2: value = p[2].decode(encoding='utf-8') else: value = p[2] self.builder.set_created_date(self.document, value) except CardinalityError: self.more_than_one_error('Created', p.lineno(1)) def p_created_2(self, p): """created : CREATED error""" self.error = True msg = ERROR_MESSAGES['CREATED_VALUE_TYPE'].format(p.lineno(1)) self.logger.log(msg) def p_entity_1(self, p): """entity : TOOL_VALUE """ try: if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] p[0] = self.builder.build_tool(self.document, value) except SPDXValueError: msg = ERROR_MESSAGES['TOOL_VALUE'].format(p[1], p.lineno(1)) self.logger.log(msg) self.error = True p[0] = None def p_entity_2(self, p): """entity : ORG_VALUE """ try: if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] p[0] = self.builder.build_org(self.document, value) except SPDXValueError: msg = ERROR_MESSAGES['ORG_VALUE'].format(p[1], p.lineno(1)) self.logger.log(msg) self.error = True p[0] = None def p_entity_3(self, p): """entity : PERSON_VALUE """ try: if six.PY2: value = p[1].decode(encoding='utf-8') else: value = p[1] p[0] = self.builder.build_person(self.document, value) except SPDXValueError: msg = ERROR_MESSAGES['PERSON_VALUE'].format(p[1], p.lineno(1)) self.logger.log(msg) self.error = True p[0] = None def p_error(self, p): pass def build(self, **kwargs): self.lex = Lexer() self.lex.build(reflags=re.UNICODE) self.yacc = yacc.yacc(module=self, **kwargs) def parse(self, text): self.document = document.Document() self.error = False self.yacc.parse(text, lexer=self.lex) # FIXME: this state does not make sense self.builder.reset() validation_messages = [] # Report extra errors if self.error is False otherwise there will be # redundent messages if (not self.error) and (not self.document.validate(validation_messages)): for msg in validation_messages: self.logger.log(msg) self.error = True return self.document, self.error
def build(self, **kwargs): self.lex = Lexer() self.lex.build(reflags=re.UNICODE) self.yacc = yacc.yacc(module=self, **kwargs)
def __init__(self): self.l = Lexer() self.l.build()
class TestLexer(object): def __init__(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-1.2 # Comment. DataLicense: CC0-1.0 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper( self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-1.2', 2) self.token_assert_helper( self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper( self.l.token(), 'DOC_COMMENT', 'DocumentComment', 5) self.token_assert_helper( self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 5) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper( self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper(self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 2) self.token_assert_helper(self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 2) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 3) self.token_assert_helper(self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 3) def token_assert_helper(self, token, type, value, line): assert token.type == type assert token.value == value assert token.lineno == line
class TestLexer(TestCase): maxDiff = None def setUp(self): self.l = Lexer() self.l.build() def test_document(self): data = ''' SPDXVersion: SPDX-2.1 # Comment. DataLicense: CC0-1.0 DocumentName: Sample_Document-V2.1 SPDXID: SPDXRef-DOCUMENT DocumentNamespace: https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301 DocumentComment: <text>This is a sample spreadsheet</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'DOC_VERSION', 'SPDXVersion', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDX-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_LICENSE', 'DataLicense', 4) self.token_assert_helper(self.l.token(), 'LINE', 'CC0-1.0', 4) self.token_assert_helper(self.l.token(), 'DOC_NAME', 'DocumentName', 5) self.token_assert_helper(self.l.token(), 'LINE', 'Sample_Document-V2.1', 5) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DOCUMENT', 6) self.token_assert_helper(self.l.token(), 'DOC_NAMESPACE', 'DocumentNamespace', 7) self.token_assert_helper(self.l.token(), 'LINE', 'https://spdx.org/spdxdocs/spdx-example-444504E0-4F89-41D3-9A0C-0305E82C3301', 7) self.token_assert_helper(self.l.token(), 'DOC_COMMENT', 'DocumentComment', 8) self.token_assert_helper(self.l.token(), 'TEXT', '<text>This is a sample spreadsheet</text>', 8) def test_external_document_references(self): data = ''' ExternalDocumentRef:DocumentRef-spdx-tool-2.1 http://spdx.org/spdxdocs/spdx-tools-v2.1-3F2504E0-4F89-41D3-9A0C-0305E82C3301 SHA1: d6a770ba38583ed4bb4525bd96e50461655d2759 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF', 'ExternalDocumentRef', 2) self.token_assert_helper(self.l.token(), 'DOC_REF_ID', 'DocumentRef-spdx-tool-2.1', 2) self.token_assert_helper(self.l.token(), 'DOC_URI', 'http://spdx.org/spdxdocs/spdx-tools-v2.1-3F25' '04E0-4F89-41D3-9A0C-0305E82C3301', 2) self.token_assert_helper(self.l.token(), 'EXT_DOC_REF_CHKSUM', 'SHA1: ' 'd6a770ba38583ed4bb4525bd96e50461655d2759', 2) def test_creation_info(self): data = ''' ## Creation Information Creator: Person: Gary O'Neall Creator: Organization: Source Auditor Inc. Creator: Tool: SourceAuditor-V1.2 Created: 2010-02-03T00:00:00Z CreatorComment: <text>This is an example of an SPDX spreadsheet format</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 3) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Gary O'Neall", 3) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 4) self.token_assert_helper(self.l.token(), 'ORG_VALUE', 'Organization: Source Auditor Inc.', 4) self.token_assert_helper(self.l.token(), 'CREATOR', 'Creator', 5) self.token_assert_helper(self.l.token(), 'TOOL_VALUE', 'Tool: SourceAuditor-V1.2', 5) self.token_assert_helper(self.l.token(), 'CREATED', 'Created', 6) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-03T00:00:00Z', 6) def test_review_info(self): data = ''' Reviewer: Person: Joe Reviewer ReviewDate: 2010-02-10T00:00:00Z ReviewComment: <text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'REVIEWER', 'Reviewer', 2) self.token_assert_helper(self.l.token(), 'PERSON_VALUE', "Person: Joe Reviewer", 2) self.token_assert_helper(self.l.token(), 'REVIEW_DATE', 'ReviewDate', 3) self.token_assert_helper(self.l.token(), 'DATE', '2010-02-10T00:00:00Z', 3) self.token_assert_helper(self.l.token(), 'REVIEW_COMMENT', 'ReviewComment', 4) self.token_assert_helper(self.l.token(), 'TEXT', '''<text>This is just an example. Some of the non-standard licenses look like they are actually BSD 3 clause licenses</text>''', 4) def test_pacakage(self): data = ''' SPDXID: SPDXRef-Package FilesAnalyzed: False PackageChecksum: SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12 PackageVerificationCode: 4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt) ExternalRef: SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*: ExternalRefComment: <text>Some comment about the package.</text> ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'SPDX_ID', 'SPDXID', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Package', 2) self.token_assert_helper(self.l.token(), 'PKG_FILES_ANALYZED', 'FilesAnalyzed', 3) self.token_assert_helper(self.l.token(), 'LINE', 'False', 3) self.token_assert_helper(self.l.token(), 'PKG_CHKSUM', 'PackageChecksum', 4) self.token_assert_helper(self.l.token(), 'CHKSUM', 'SHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12', 4) self.token_assert_helper(self.l.token(), 'PKG_VERF_CODE', 'PackageVerificationCode', 5) self.token_assert_helper(self.l.token(), 'LINE', '4e3211c67a2d28fced849ee1bb76e7391b93feba (SpdxTranslatorSpdx.rdf, SpdxTranslatorSpdx.txt)', 5) self.token_assert_helper(self.l.token(), 'PKG_EXT_REF', 'ExternalRef', 6) self.token_assert_helper(self.l.token(), 'LINE', 'SECURITY cpe23Type cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:', 6) self.token_assert_helper(self.l.token(), 'PKG_EXT_REF_COMMENT', 'ExternalRefComment', 7) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some comment about the package.</text>', 7) def test_unknown_tag(self): data = ''' SomeUnknownTag: SomeUnknownValue ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'UNKNOWN_TAG', 'SomeUnknownTag', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SomeUnknownValue', 2) def test_snippet(self): data = ''' SnippetSPDXID: SPDXRef-Snippet SnippetLicenseComments: <text>Some lic comment.</text> SnippetCopyrightText: <text>Some cr text.</text> SnippetComment: <text>Some snippet comment.</text> SnippetName: from linux kernel SnippetFromFileSPDXID: SPDXRef-DoapSource SnippetLicenseConcluded: Apache-2.0 LicenseInfoInSnippet: Apache-2.0 ''' self.l.input(data) self.token_assert_helper(self.l.token(), 'SNIPPET_SPDX_ID', 'SnippetSPDXID', 2) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-Snippet', 2) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_COMMENT', 'SnippetLicenseComments', 3) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some lic comment.</text>', 3) self.token_assert_helper(self.l.token(), 'SNIPPET_CR_TEXT', 'SnippetCopyrightText', 4) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some cr text.</text>', 4) self.token_assert_helper(self.l.token(), 'SNIPPET_COMMENT', 'SnippetComment', 5) self.token_assert_helper(self.l.token(), 'TEXT', '<text>Some snippet comment.</text>', 5) self.token_assert_helper(self.l.token(), 'SNIPPET_NAME', 'SnippetName', 6) self.token_assert_helper(self.l.token(), 'LINE', 'from linux kernel', 6) self.token_assert_helper(self.l.token(), 'SNIPPET_FILE_SPDXID', 'SnippetFromFileSPDXID', 7) self.token_assert_helper(self.l.token(), 'LINE', 'SPDXRef-DoapSource', 7) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_CONC', 'SnippetLicenseConcluded', 8) self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 8) self.token_assert_helper(self.l.token(), 'SNIPPET_LICS_INFO', 'LicenseInfoInSnippet', 9) self.token_assert_helper(self.l.token(), 'LINE', 'Apache-2.0', 9) def token_assert_helper(self, token, ttype, value, line): assert token.type == ttype assert token.value == value assert token.lineno == line