def add_package(package, parent=None): """ Function to recursively add a package and it's deps""" spdxpackage = SpdxPackage(name=package.package_name, version=package.version) spdxpackage.spdx_id = f'SPDXRef-{id_count[0]}' id_count[0] += 1 spdxpackage.homepage = SPDXNone() spdxpackage.cr_text = NoAssert() spdxpackage.download_location = UnKnown() spdxpackage.files_analyzed = False spdxpackage.conc_lics = NoAssert() spdxpackage.license_declared = NoAssert() spdxpackage.licenses_from_files = [NoAssert()] # if we have a parent be sure to list the relationship if parent != None: spdxpackage.add_relationship( Relationship(spdxpackage, RelationshipOptions.PACKAGE_OF, parent)) # go through the same process for depenedencies for dep in package.dependencies: add_package(dep, parent=spdxpackage) # finally add it to the document doc.add_package(spdxpackage)
def set_package_info(self, package): # Use a set of unique copyrights for the package. package.name = basename(self.path_or_file) if self.path_or_file == ".": package.name = os.getcwd().split("/")[-1] if self.file_to_scan: package.name = "{0}/{1}".format(basename(self.path_or_file), basename(self.file_to_scan)) package.check_sum = Algorithm("SHA1", self.get_package_checksum()) package.homepage = SPDXNone() package.verif_code = self.get_package_verification_code() package.source_info = SPDXNone() package.conc_lics = NoAssert() package.license_declared = NoAssert() package.cr_text = SPDXNone()
def create_package(package_name: str, licenses_dict: dict, p_id: int) -> package.Package: logging.debug(f"Creating SBOM Package section") pkg = package.Package(name=package_name, spdx_id=f"SPDXRef-PACKAGE-{p_id}", download_location=extra_conf.get( 'package_location', NoAssert())) pkg.check_sum = Algorithm(identifier="SHA1", value=extra_conf.get('package_sha1', NoAssert())) pkg.license_declared = get_license_obj( extra_conf.get('package_license_identifier'), licenses_dict) pkg.conc_lics = get_license_obj(extra_conf.get('package_conc_licenses'), licenses_dict) pkg.cr_text = extra_conf.get('package_copyright_text', NoAssert()) pkg.supplier = creationinfo.Person( extra_conf.get('package_supplier', SPDXNone()), extra_conf.get('package_supplier_email', SPDXNone())) logging.debug(f"Finished SBOM package section") return pkg
def determine_spdx_value(value: Optional[str]) -> Union[str, UnKnown, SPDXNone]: """Determines the correct SPDX value. Args: value: a value Returns: correct SPDX value a string, UnKnown or SPDXNone """ if not value: return SPDXNone() if value == UNKNOWN: return UnKnown() return value
def serialize(self, packages: List[Package]) -> str: doc = Document(name="Translated SBOM", namespace=SPDXNone(), spdx_id="SPDXRef-DOCUMENT") doc.version = Version(2, 1) doc.comment = 'Translated with Decoder Ring' doc.data_license = License.from_identifier('CC0-1.0') doc.creation_info.add_creator(Tool("Decoder Ring")) doc.creation_info.set_created_now() # form SPDX ids within the document just use a simple counter id_count = [ 1 ] # python closure trick, it needs to be a mutable object like a list for closure to work def add_package(package, parent=None): """ Function to recursively add a package and it's deps""" spdxpackage = SpdxPackage(name=package.package_name, version=package.version) spdxpackage.spdx_id = f'SPDXRef-{id_count[0]}' id_count[0] += 1 spdxpackage.homepage = SPDXNone() spdxpackage.cr_text = NoAssert() spdxpackage.download_location = UnKnown() spdxpackage.files_analyzed = False spdxpackage.conc_lics = NoAssert() spdxpackage.license_declared = NoAssert() spdxpackage.licenses_from_files = [NoAssert()] # if we have a parent be sure to list the relationship if parent != None: spdxpackage.add_relationship( Relationship(spdxpackage, RelationshipOptions.PACKAGE_OF, parent)) # go through the same process for depenedencies for dep in package.dependencies: add_package(dep, parent=spdxpackage) # finally add it to the document doc.add_package(spdxpackage) for package in packages: add_package(package) out = io.StringIO() write_document(doc, out) return out.getvalue()
doc.data_license = License.from_identifier('CC0-1.0') doc.creation_info.add_creator(Person('Alice', '*****@*****.**')) doc.creation_info.set_created_now() review = Review(Person('Joe', None)) review.set_review_date_now() review.comment = 'Joe reviewed this document' doc.add_review(review) # File testfile1 = File('TestFile1') testfile1.type = FileType.BINARY testfile1.comment = 'This is a test file.' testfile1.chk_sum = Algorithm('SHA1', 'c537c5d99eca5333f23491d47ededd083fefb7ad') testfile1.conc_lics = License.from_identifier('BSD-2-Clause') testfile1.add_lics(License.from_identifier('BSD-2-Clause')) testfile1.copyright = SPDXNone() testfile1.add_artifact('name', 'TagWriteTest') testfile1.add_artifact('home', UnKnown()) testfile1.add_artifact('uri', 'http://tagwritetest.test') testfile2 = File('TestFile2') testfile2.type = FileType.SOURCE testfile2.comment = 'This is a test file.' testfile2.chk_sum = Algorithm('SHA1', 'bb154f28d1cf0646ae21bb0bec6c669a2b90e113') testfile2.conc_lics = License.from_identifier('Apache-2.0') testfile2.add_lics(License.from_identifier('Apache-2.0')) testfile2.copyright = NoAssert() # Package package = Package()
def write_formatted_output(scanners, files_count, version, notice, scanned_files, format, options, input, output_file, _echo, _save): """ Save scan results to file or screen. """ if format == 'html': for template_chunk in as_template(scanned_files, files_count, output_file): try: output_file.write(template_chunk) except Exception as e: extra_context = 'ERROR: Failed to write output to HTML for: ' + repr( template_chunk) _echo(extra_context, fg='red') _save(extra_context + '\n', output_file) e.args += (extra_context, ) raise e elif format == 'html-app': output_file.write(as_html_app(input, output_file)) try: create_html_app_assets(scanned_files, output_file) except HtmlAppAssetCopyWarning: _echo('\nHTML app creation skipped when printing to stdout.', fg='yellow') _save('\nHTML app creation skipped when printing to stdout.', output_file) except HtmlAppAssetCopyError: _echo('\nFailed to create HTML app.', fg='red') _save('\nFailed to create HTML app.', output_file) elif format == 'json' or format == 'json-pp': import simplejson as json meta = OrderedDict() meta['scancode_notice'] = notice meta['scancode_version'] = version meta['scancode_options'] = options meta['files_count'] = files_count meta['files'] = scanned_files if format == 'json-pp': output_file.write( unicode( json.dumps(meta, indent=2 * ' ', iterable_as_array=True, encoding='utf-8'))) else: output_file.write( unicode( json.dumps(meta, separators=(',', ':'), iterable_as_array=True, encoding='utf-8'))) output_file.write('\n') elif format in ('spdx-tv', 'spdx-rdf'): from spdx.checksum import Algorithm from spdx.creationinfo import Tool from spdx.document import Document, License from spdx.file import File from spdx.package import Package from spdx.utils import NoAssert from spdx.utils import SPDXNone from spdx.version import Version input = abspath(input) if os.path.isdir(input): input_path = input else: input_path = os.path.dirname(input) doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) doc.creation_info.add_creator(Tool('ScanCode ' + version)) doc.creation_info.set_created_now() doc.package = Package(os.path.basename(input_path), NoAssert()) # Use a set of unique copyrights for the package. doc.package.cr_text = set() all_files_have_no_license = True all_files_have_no_copyright = True for file_data in scanned_files: # Construct the absolute path in case we need to access the file # to calculate its SHA1. file_entry = File(os.path.join(input_path, file_data.get('path'))) file_sha1 = file_data.get('sha1') if not file_sha1: if os.path.isfile(file_entry.name): # Calculate the SHA1 in case it is missing, e.g. for empty files. file_sha1 = file_entry.calc_chksum() else: # Skip directories. continue # Restore the relative file name as that is what we want in # SPDX output (with explicit leading './'). file_entry.name = './' + file_data.get('path') file_entry.chk_sum = Algorithm('SHA1', file_sha1) file_licenses = file_data.get('licenses') if file_licenses: all_files_have_no_license = False for file_license in file_licenses: spdx_id = file_license.get('spdx_license_key') if spdx_id: spdx_license = License.from_identifier(spdx_id) else: license_key = 'LicenseRef-' + file_license.get('key') spdx_license = License(file_license.get('short_name'), license_key) # Add licenses in the order they appear in the file. Maintaining the order # might be useful for provenance purposes. file_entry.add_lics(spdx_license) doc.package.add_lics_from_file(spdx_license) else: if file_licenses == None: all_files_have_no_license = False spdx_license = NoAssert() else: spdx_license = SPDXNone() file_entry.add_lics(spdx_license) file_entry.conc_lics = NoAssert() file_copyrights = file_data.get('copyrights') if file_copyrights: all_files_have_no_copyright = False file_entry.copyright = [] for file_copyright in file_copyrights: file_entry.copyright.extend( file_copyright.get('statements')) doc.package.cr_text.update(file_entry.copyright) # Create a text of copyright statements in the order they appear in the file. # Maintaining the order might be useful for provenance purposes. file_entry.copyright = '\n'.join(file_entry.copyright) + '\n' else: if file_copyrights == None: all_files_have_no_copyright = False spdx_copyright = NoAssert() else: spdx_copyright = SPDXNone() file_entry.copyright = spdx_copyright doc.package.add_file(file_entry) if len(doc.package.files) == 0: if format == 'spdx-tv': output_file.write("# No results for package '{}'.\n".format( doc.package.name)) else: output_file.write( "<!-- No results for package '{}'. -->\n".format( doc.package.name)) return # Remove duplicate licenses from the list for the package. unique_licenses = set(doc.package.licenses_from_files) if len(doc.package.licenses_from_files) == 0: if all_files_have_no_license: doc.package.licenses_from_files = [SPDXNone()] else: doc.package.licenses_from_files = [NoAssert()] else: # List license identifiers alphabetically for the package. doc.package.licenses_from_files = sorted( unique_licenses, key=lambda x: x.identifier) if len(doc.package.cr_text) == 0: if all_files_have_no_copyright: doc.package.cr_text = SPDXNone() else: doc.package.cr_text = NoAssert() else: # Create a text of alphabetically sorted copyright statements for the package. doc.package.cr_text = '\n'.join(sorted(doc.package.cr_text)) + '\n' doc.package.verif_code = doc.package.calc_verif_code() doc.package.license_declared = NoAssert() doc.package.conc_lics = NoAssert() # As the spdx-tools package can only write the document to a "str" file but ScanCode provides a "unicode" file, # write to a "str" buffer first and then manually write the value to a "unicode" file. from StringIO import StringIO str_buffer = StringIO() if format == 'spdx-tv': from spdx.writers.tagvalue import write_document write_document(doc, str_buffer) else: from spdx.writers.rdf import write_document write_document(doc, str_buffer) output_file.write(str_buffer.getvalue()) else: raise Exception('Unknown format')
def write_spdx(output_file, files, tool_name, tool_version, notice, input_file, as_tagvalue=True): """ Write scan output as SPDX Tag/value or RDF. """ _patch_license_list() absinput = abspath(input_file) if isdir(absinput): input_path = absinput else: input_path = dirname(absinput) doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) doc.comment = notice tool_name = tool_name or 'ScanCode' doc.creation_info.add_creator(Tool(tool_name + ' ' + tool_version)) doc.creation_info.set_created_now() package = doc.package = Package( name=basename(input_path), download_location=NoAssert() ) # Use a set of unique copyrights for the package. package.cr_text = set() all_files_have_no_license = True all_files_have_no_copyright = True # FIXME: this should walk the codebase instead!!! for file_data in files: # Skip directories. if file_data.get('type') != 'file': continue # Set a relative file name as that is what we want in # SPDX output (with explicit leading './'). name = './' + file_data.get('path') file_entry = File( name=name, chk_sum=Algorithm('SHA1', file_data.get('sha1') or '') ) file_licenses = file_data.get('licenses') if file_licenses: all_files_have_no_license = False for file_license in file_licenses: spdx_id = file_license.get('spdx_license_key') if spdx_id: spdx_license = License.from_identifier(spdx_id) else: license_key = file_license.get('key') licenseref_id = 'LicenseRef-scancode-' + license_key spdx_license = ExtractedLicense(licenseref_id) spdx_license.name = file_license.get('short_name') comment = ('See details at https://github.com/nexB/scancode-toolkit' '/blob/develop/src/licensedcode/data/licenses/%s.yml\n' % license_key) spdx_license.comment = comment text = file_license.get('matched_text') # always set some text, even if we did not extract the matched text if not text: text = comment spdx_license.text = text doc.add_extr_lic(spdx_license) # Add licenses in the order they appear in the file. Maintaining the order # might be useful for provenance purposes. file_entry.add_lics(spdx_license) package.add_lics_from_file(spdx_license) elif file_licenses is None: all_files_have_no_license = False file_entry.add_lics(NoAssert()) else: file_entry.add_lics(SPDXNone()) file_entry.conc_lics = NoAssert() file_copyrights = file_data.get('copyrights') if file_copyrights: all_files_have_no_copyright = False file_entry.copyright = [] for file_copyright in file_copyrights: file_entry.copyright.append(file_copyright.get('value')) package.cr_text.update(file_entry.copyright) # Create a text of copyright statements in the order they appear in the file. # Maintaining the order might be useful for provenance purposes. file_entry.copyright = '\n'.join(file_entry.copyright) + '\n' elif file_copyrights is None: all_files_have_no_copyright = False file_entry.copyright = NoAssert() else: file_entry.copyright = SPDXNone() package.add_file(file_entry) if len(package.files) == 0: if as_tagvalue: msg = "# No results for package '{}'.\n".format(package.name) else: # rdf msg = "<!-- No results for package '{}'. -->\n".format(package.name) output_file.write(msg) # Remove duplicate licenses from the list for the package. unique_licenses = {(l.identifier, l.full_name): l for l in package.licenses_from_files} unique_licenses = list(unique_licenses.values()) if not len(package.licenses_from_files): if all_files_have_no_license: package.licenses_from_files = [SPDXNone()] else: package.licenses_from_files = [NoAssert()] else: # List license identifiers alphabetically for the package. package.licenses_from_files = sorted(unique_licenses, key=lambda x: x.identifier) if len(package.cr_text) == 0: if all_files_have_no_copyright: package.cr_text = SPDXNone() else: package.cr_text = NoAssert() else: # Create a text of alphabetically sorted copyright # statements for the package. package.cr_text = '\n'.join(sorted(package.cr_text)) + '\n' package.verif_code = doc.package.calc_verif_code() package.license_declared = NoAssert() package.conc_lics = NoAssert() # The spdx-tools write_document returns either: # - unicode for tag values # - UTF8-encoded bytes for rdf because somehow the rdf and xml # libraries do the encoding and do not return text but bytes # The file passed by ScanCode for output is opened in text mode Therefore in # one case we do need to deal with bytes and decode before writing (rdf) and # in the other case we deal with text all the way. if package.files: if as_tagvalue: from spdx.writers.tagvalue import write_document # NOQA else: from spdx.writers.rdf import write_document # NOQA if as_tagvalue: # unicode text everywhere spdx_output = StringIO() else: # rdf as utf-encoded bytes on Py2 spdx_output = BytesIO() write_document(doc, spdx_output, validate=False) result = spdx_output.getvalue() if as_tagvalue: # unicode text everywhere pass else: # rdf as utf-encoded bytes on Py2 result = result.decode('utf-8') output_file.write(result)
def write_spdx(version, notice, scanned_files, input, output_file, as_tagvalue=True): """ Write scan output formatted as SPDX Tag/value or RDF. """ absinput = abspath(input) if os.path.isdir(absinput): input_path = absinput else: input_path = os.path.dirname(absinput) doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) doc.comment = notice doc.creation_info.add_creator(Tool('ScanCode ' + version)) doc.creation_info.set_created_now() package = doc.package = Package(name=os.path.basename(input_path), download_location=NoAssert()) # Use a set of unique copyrights for the package. package.cr_text = set() all_files_have_no_license = True all_files_have_no_copyright = True for file_data in scanned_files: # Construct the absolute path in case we need to access the file # to calculate its SHA1. file_entry = File(os.path.join(input_path, file_data.get('path'))) file_sha1 = file_data.get('sha1') if not file_sha1: if os.path.isfile(file_entry.name): # Calculate the SHA1 in case it is missing, e.g. for empty files. file_sha1 = file_entry.calc_chksum() else: # Skip directories. continue # Restore the relative file name as that is what we want in # SPDX output (with explicit leading './'). file_entry.name = './' + file_data.get('path') file_entry.chk_sum = Algorithm('SHA1', file_sha1) file_licenses = file_data.get('licenses') if file_licenses: all_files_have_no_license = False for file_license in file_licenses: spdx_id = file_license.get('spdx_license_key') if spdx_id: # spdx_id = spdx_id.rstrip('+') spdx_license = License.from_identifier(spdx_id) else: license_key = file_license.get('key') # FIXME: we should prefix this with ScanCode- licenseref_id = 'LicenseRef-' + license_key spdx_license = ExtractedLicense(licenseref_id) spdx_license.name = file_license.get('short_name') comment = 'See details at https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/%s.yml\n' % license_key spdx_license.comment = comment text = file_license.get('matched_text') # always set some text, even if we did not extract the matched text if not text: text = comment spdx_license.text = text doc.add_extr_lic(spdx_license) # Add licenses in the order they appear in the file. Maintaining the order # might be useful for provenance purposes. file_entry.add_lics(spdx_license) package.add_lics_from_file(spdx_license) elif file_licenses is None: all_files_have_no_license = False file_entry.add_lics(NoAssert()) else: file_entry.add_lics(SPDXNone()) file_entry.conc_lics = NoAssert() file_copyrights = file_data.get('copyrights') if file_copyrights: all_files_have_no_copyright = False file_entry.copyright = [] for file_copyright in file_copyrights: file_entry.copyright.extend(file_copyright.get('statements')) package.cr_text.update(file_entry.copyright) # Create a text of copyright statements in the order they appear in the file. # Maintaining the order might be useful for provenance purposes. file_entry.copyright = '\n'.join(file_entry.copyright) + '\n' elif file_copyrights is None: all_files_have_no_copyright = False file_entry.copyright = NoAssert() else: file_entry.copyright = SPDXNone() package.add_file(file_entry) if len(package.files) == 0: if as_tagvalue: output_file.write("# No results for package '{}'.\n".format( package.name)) else: output_file.write("<!-- No results for package '{}'. -->\n".format( package.name)) # Remove duplicate licenses from the list for the package. unique_licenses = set(package.licenses_from_files) if not len(package.licenses_from_files): if all_files_have_no_license: package.licenses_from_files = [SPDXNone()] else: package.licenses_from_files = [NoAssert()] else: # List license identifiers alphabetically for the package. package.licenses_from_files = sorted(unique_licenses, key=lambda x: x.identifier) if len(package.cr_text) == 0: if all_files_have_no_copyright: package.cr_text = SPDXNone() else: package.cr_text = NoAssert() else: # Create a text of alphabetically sorted copyright # statements for the package. package.cr_text = '\n'.join(sorted(package.cr_text)) + '\n' package.verif_code = doc.package.calc_verif_code() package.license_declared = NoAssert() package.conc_lics = NoAssert() if as_tagvalue: from spdx.writers.tagvalue import write_document else: from spdx.writers.rdf import write_document # The spdx-tools write_document returns either: # - unicode for tag values # - UTF8-encoded bytes for rdf because somehow the rd and xml # libraries do the encoding # The file passed by ScanCode for output is alwasy opened in binary # mode and needs to receive UTF8-encoded bytes. # Therefore in one case we do nothing (rdf) and in the other case we # encode to UTF8 bytes. from StringIO import StringIO spdx_output = StringIO() write_document(doc, spdx_output, validate=True) result = spdx_output.getvalue() if as_tagvalue: result = result.encode('utf-8') output_file.write(result)
def write_spdx( output_file, files, tool_name, tool_version, notice, package_name='', download_location=NoAssert(), as_tagvalue=True, spdx_version=(2, 2), with_notice_text=False, ): """ Write scan output as SPDX Tag/value to ``output_file`` file-like object using the ``files`` list of scanned file data. Write as RDF XML if ``as_tagvalue`` is False. Use the ``notice`` string as a notice included in a document comment. Include the ``tool_name`` and ``tool_version`` to indicate which tool is producing this SPDX document. Use ``package_name`` as a Package name and as a namespace prefix base. """ as_rdf = not as_tagvalue _patch_license_list() ns_prefix = '_'.join(package_name.lower().split()) comment = notice + f'\nSPDX License List: {scancode_config.spdx_license_list_version}' doc = Document( version=Version(*spdx_version), data_license=License.from_identifier('CC0-1.0'), comment=notice, namespace=f'http://spdx.org/spdxdocs/{ns_prefix}-{uuid.uuid4()}', license_list_version=scancode_config.spdx_license_list_version, name='SPDX Document created by ScanCode Toolkit') tool_name = tool_name or 'ScanCode' doc.creation_info.add_creator(Tool(f'{tool_name} {tool_version}')) doc.creation_info.set_created_now() package_id = '001' package = doc.package = Package( name=package_name, download_location=download_location, spdx_id=f'SPDXRef-{package_id}', ) # Use a set of unique copyrights for the package. package.cr_text = set() all_files_have_no_license = True all_files_have_no_copyright = True # FIXME: this should walk the codebase instead!!! for sid, file_data in enumerate(files, 1): # Skip directories. if file_data.get('type') != 'file': continue # Set a relative file name as that is what we want in # SPDX output (with explicit leading './'). name = './' + file_data.get('path') file_entry = File(spdx_id=f'SPDXRef-{sid}', name=name, chk_sum=Algorithm('SHA1', file_data.get('sha1') or '')) file_licenses = file_data.get('licenses') if file_licenses: all_files_have_no_license = False for file_license in file_licenses: license_key = file_license.get('key') spdx_id = file_license.get('spdx_license_key') if not spdx_id: spdx_id = f'LicenseRef-scancode-{license_key}' is_license_ref = spdx_id.lower().startswith('licenseref-') if not is_license_ref: spdx_license = License.from_identifier(spdx_id) else: spdx_license = ExtractedLicense(spdx_id) spdx_license.name = file_license.get('short_name') # FIXME: replace this with the licensedb URL comment = ( f'See details at https://github.com/nexB/scancode-toolkit' f'/blob/develop/src/licensedcode/data/licenses/{license_key}.yml\n' ) spdx_license.comment = comment text = file_license.get('matched_text') # always set some text, even if we did not extract the # matched text if not text: text = comment spdx_license.text = text doc.add_extr_lic(spdx_license) # Add licenses in the order they appear in the file. Maintaining # the order might be useful for provenance purposes. file_entry.add_lics(spdx_license) package.add_lics_from_file(spdx_license) elif file_licenses is None: all_files_have_no_license = False file_entry.add_lics(NoAssert()) else: file_entry.add_lics(SPDXNone()) file_entry.conc_lics = NoAssert() file_copyrights = file_data.get('copyrights') if file_copyrights: all_files_have_no_copyright = False file_entry.copyright = [] for file_copyright in file_copyrights: file_entry.copyright.append(file_copyright.get('copyright')) package.cr_text.update(file_entry.copyright) # Create a text of copyright statements in the order they appear in # the file. Maintaining the order might be useful for provenance # purposes. file_entry.copyright = '\n'.join(file_entry.copyright) + '\n' elif file_copyrights is None: all_files_have_no_copyright = False file_entry.copyright = NoAssert() else: file_entry.copyright = SPDXNone() package.add_file(file_entry) if len(package.files) == 0: if as_tagvalue: msg = "# No results for package '{}'.\n".format(package.name) else: # rdf msg = "<!-- No results for package '{}'. -->\n".format( package.name) output_file.write(msg) # Remove duplicate licenses from the list for the package. unique_licenses = {l.identifier: l for l in package.licenses_from_files} unique_licenses = list(unique_licenses.values()) if not len(package.licenses_from_files): if all_files_have_no_license: package.licenses_from_files = [SPDXNone()] else: package.licenses_from_files = [NoAssert()] else: # List license identifiers alphabetically for the package. package.licenses_from_files = sorted( unique_licenses, key=lambda x: x.identifier, ) if len(package.cr_text) == 0: if all_files_have_no_copyright: package.cr_text = SPDXNone() else: package.cr_text = NoAssert() else: # Create a text of alphabetically sorted copyright # statements for the package. package.cr_text = '\n'.join(sorted(package.cr_text)) + '\n' package.verif_code = doc.package.calc_verif_code() package.license_declared = NoAssert() package.conc_lics = NoAssert() # The spdx-tools write_document returns either: # - unicode for tag values # - UTF8-encoded bytes for rdf because somehow the rdf and xml # libraries do the encoding and do not return text but bytes # The file passed by ScanCode for output is opened in text mode Therefore in # one case we do need to deal with bytes and decode before writing (rdf) and # in the other case we deal with text all the way. if package.files: if as_tagvalue: from spdx.writers.tagvalue import write_document # NOQA elif as_rdf: from spdx.writers.rdf import write_document # NOQA if as_tagvalue: spdx_output = StringIO() elif as_rdf: # rdf is utf-encoded bytes spdx_output = BytesIO() write_document(doc, spdx_output, validate=False) result = spdx_output.getvalue() if as_rdf: # rdf is utf-encoded bytes result = result.decode('utf-8') output_file.write(result)
def create_spdx_document(self): """ Write identifier scan results as SPDX Tag/value or RDF. """ logging.basicConfig(level=logging.INFO) logging.info("Creating spdx document") self.get_output_file() self.spdx_document = Document( version=Version(2, 1), data_license=License.from_identifier( self.code_extra_params["lic_identifier"]), ) self.set_creation_info() if isdir(self.path_or_file): input_path = self.path_or_file else: input_path = dirname(self.path_or_file) package = self.spdx_document.package = Package( download_location=NoAssert(), version=self.get_package_version()) self.set_package_info(package) all_files_have_no_license = True all_files_have_no_copyright = True file_license_list = [] file_license_ids = [] if is_dir(self.path_or_file): for idx, file_data in enumerate(self.id_scan_results): file_data_instance = open(file_data["FileName"], "r") if not should_skip_file(file_data["FileName"], self.output_file_name): name = file_data["FileName"].replace( self.path_or_file, ".") file_entry = File( name=name, chk_sum=Algorithm( "SHA1", get_file_hash(file_data["FileName"]) or ""), ) spdx_license = None if self.doc_type == TAG_VALUE: spdx_license = License.from_identifier( file_data["SPDXID"]) else: licenseref_id = "SPDXID-Doc-Generator-" + file_data[ "SPDXID"] file_license_ids.append(licenseref_id) if licenseref_id in file_license_ids: spdx_license = ExtractedLicense(licenseref_id) spdx_license.name = NoAssert() comment = "N/A" spdx_license.comment = comment text = NoAssert() if not text: text = comment spdx_license.text = text self.spdx_document.add_extr_lic(spdx_license) package.add_lics_from_file(spdx_license) file_entry.add_lics(spdx_license) file_license_list.append(spdx_license) file_entry.conc_lics = NoAssert() file_entry.copyright = SPDXNone() file_entry.spdx_id = self.code_extra_params[ "file_ref"].format(idx + 1) package.add_file(file_entry) if self.doc_type == TAG_VALUE: for spdx_license in list(set(file_license_list)): package.add_lics_from_file(spdx_license) if len(package.files) == 0: if self.doc_type == TAG_VALUE: self.output_file.write( "# No results for package '{}'.\n".format(package.name)) else: self.output_file.write( "<!-- No results for package '{}'. -->\n".format( package.name)) if self.doc_type == TAG_VALUE: from spdx.writers.tagvalue import write_document # NOQA else: from spdx.writers.rdf import write_document # NOQA if package.files: spdx_output = io.StringIO() if self.doc_type == TAG_VALUE: write_document(self.spdx_document, spdx_output, validate=False) logging.info("SPDX Tag-Value Document created successfully.") else: # spdx_output = io.BytesIO() write_document(self.spdx_document, spdx_output, validate=False) logging.info("SPDX RDF Document created successfully.") result = spdx_output.getvalue() if self.doc_type == TAG_VALUE: result = result.encode("utf-8") self.output_file.write(result)
def create_files(scope_token: str, licenses_dict: dict) -> tuple: # filter set to contain only a single of SPDXNone and NOASSERT def filter_none_types(in_set): out_set = set() no_assert_in_set = False spdx_none_int_set = False for ent in in_set: ent_is_no_assert = isinstance(ent, NoAssert) ent_is_spdx_none = isinstance(ent, SPDXNone) if not no_assert_in_set and ent_is_no_assert: no_assert_in_set = True out_set.add(ent) elif not spdx_none_int_set and ent_is_spdx_none: spdx_none_int_set = True out_set.add(ent) elif not ent_is_spdx_none and not ent_is_no_assert: out_set.add(ent) return out_set def create_spdx_filename(lib_name: str, l_loc: dict) -> str: path = "" locations = l_loc.get('locations') if len(locations): if len(locations) > 1: logging.warning( f"Found {len(l_loc['locations'])} locations for lib {lib_name}. Using the first one" ) location = locations[0] try: split_path = re.split('\\\\|\\|/|//', location['path']) path = f"../{split_path[-4]}/{split_path[-3]}/{split_path[-2]}/" except KeyError: logging.error(f"No path value in lib: {lib_name} ") else: logging.error(f"No locations found for lib {lib_name} ") rel_path = path + lib_name logging.debug( f"Received filename: {lib_name}. SPDX relative path: {rel_path}") return rel_path global ws_conn files = [] all_licenses_from_files = set() all_copyright_from_files = set() all_extracted_licenses_from_files = list() dd_list = ws_conn.get_due_diligence(token=scope_token) dd_dict = ws_utilities.convert_dict_list_to_dict(lst=dd_list, key_desc=('library', 'name')) libs = ws_conn.get_licenses(token=scope_token) libs_loc_list = ws_conn.get_library_location(token=scope_token) libs_loc = ws_utilities.convert_dict_list_to_dict(libs_loc_list, 'keyUuid') lib_filenames = set() for i, lib in enumerate(libs): lib_loc = libs_loc[lib['keyUuid']] spdx_filename = create_spdx_filename(lib['filename'], lib_loc) if spdx_filename not in lib_filenames: lib_filenames.add( spdx_filename ) # Tracking lib names as SPDX does not allow duplications logging.debug(f"Handling library (filename: {spdx_filename}") spdx_file = file.File(name=spdx_filename, spdx_id=f"SPDXRef-FILE-{i+1}", chk_sum=Algorithm(identifier="SHA1", value=lib['sha1'])) spdx_file.comment = lib.get('description') spdx_file.type = set_file_type(lib['type'], spdx_filename) file_licenses, extracted_licenses = handle_file_licenses( lib['licenses'], licenses_dict) spdx_file.licenses_in_file = list(file_licenses) all_licenses_from_files.update(file_licenses) all_extracted_licenses_from_files.extend(extracted_licenses) spdx_file.conc_lics = SPDXNone() file_copyrights = handle_file_copyright(lib['licenses'], lib, dd_dict) if file_copyrights: spdx_file.copyright = ', '.join(file_copyrights) all_copyright_from_files.update(file_copyrights) else: spdx_file.copyright = NoAssert() files.append(spdx_file) else: logging.warning( f"Found duplicate library: {lib['name']}, filename: {spdx_filename} ID: {lib['keyUuid']}. Skipping" ) all_licenses_from_files = filter_none_types(all_licenses_from_files) return files, all_licenses_from_files, all_copyright_from_files, all_extracted_licenses_from_files
def create(self): """ Write identifier scan results as SPDX Tag/value or RDF. """ self.get_output_file() self.spdx_document = Document( version=Version(2, 1), data_license=License.from_identifier( self.code_extra_params["lic_identifier"])) self.set_creation_info() if isdir(self.path_or_file): input_path = self.path_or_file else: input_path = dirname(self.path_or_file) package = self.spdx_document.package = Package( download_location=NoAssert(), version=self.get_package_version()) self.set_package_info(package) all_files_have_no_license = True all_files_have_no_copyright = True file_license_list = [] file_license_ids = [] bar = Bar('Writing to spdx file', max=len(self.id_scan_results)) if isPath(self.path_or_file): for idx, file_data in enumerate(self.id_scan_results): file_data_instance = open(file_data["FileName"], "r") if not shouldSkipFile(file_data["FileName"], self.output_file_name): name = file_data["FileName"].replace( self.path_or_file, '.') file_entry = File(name=name, chk_sum=Algorithm( 'SHA1', get_file_hash(file_data["FileName"]) or '')) spdx_license = None if self.doc_type == TAG_VALUE: spdx_license = License.from_identifier( file_data["SPDXID"]) else: licenseref_id = 'SPDXID-Doc-Generator-' + file_data[ "SPDXID"] file_license_ids.append(licenseref_id) if licenseref_id in file_license_ids: spdx_license = ExtractedLicense(licenseref_id) spdx_license.name = NoAssert() comment = "N/A" spdx_license.comment = comment text = NoAssert() if not text: text = comment spdx_license.text = text self.spdx_document.add_extr_lic(spdx_license) package.add_lics_from_file(spdx_license) file_entry.add_lics(spdx_license) file_license_list.append(spdx_license) file_entry.conc_lics = NoAssert() file_entry.copyright = SPDXNone() file_entry.spdx_id = self.code_extra_params[ "file_ref"].format(idx + 1) package.add_file(file_entry) bar.next() if self.doc_type == TAG_VALUE: for spdx_license in list(set(file_license_list)): package.add_lics_from_file(spdx_license) bar.finish() if len(package.files) == 0: if self.doc_type == TAG_VALUE: self.output_file.write( "# No results for package '{}'.\n".format(package.name)) else: self.output_file.write( "<!-- No results for package '{}'. -->\n".format( package.name)) if self.doc_type == TAG_VALUE: from spdx.writers.tagvalue import write_document # NOQA else: from spdx.writers.rdf import write_document # NOQA if package.files: spdx_output = io.StringIO() if self.doc_type == TAG_VALUE: write_document(self.spdx_document, spdx_output, validate=True) else: spdx_output = io.BytesIO() write_document(self.spdx_document, spdx_output, validate=True) result = spdx_output.getvalue() if self.doc_type == TAG_VALUE: result = result.encode('utf-8') self.output_file.write(result)
def write_spdx(version, notice, scanned_files, format, input, output_file): from spdx.checksum import Algorithm from spdx.creationinfo import Tool from spdx.document import Document from spdx.document import License from spdx.document import ExtractedLicense from spdx.file import File from spdx.package import Package from spdx.utils import NoAssert from spdx.utils import SPDXNone from spdx.version import Version absinput = abspath(input) if os.path.isdir(absinput): input_path = absinput else: input_path = os.path.dirname(absinput) doc = Document(Version(2, 1), License.from_identifier('CC0-1.0')) doc.comment = notice doc.creation_info.add_creator(Tool('ScanCode ' + version)) doc.creation_info.set_created_now() doc.package = Package(os.path.basename(input_path), NoAssert()) # Use a set of unique copyrights for the package. doc.package.cr_text = set() all_files_have_no_license = True all_files_have_no_copyright = True for file_data in scanned_files: # Construct the absolute path in case we need to access the file # to calculate its SHA1. file_entry = File(os.path.join(input_path, file_data.get('path'))) file_sha1 = file_data.get('sha1') if not file_sha1: if os.path.isfile(file_entry.name): # Calculate the SHA1 in case it is missing, e.g. for empty files. file_sha1 = file_entry.calc_chksum() else: # Skip directories. continue # Restore the relative file name as that is what we want in # SPDX output (with explicit leading './'). file_entry.name = './' + file_data.get('path') file_entry.chk_sum = Algorithm('SHA1', file_sha1) file_licenses = file_data.get('licenses') if file_licenses: all_files_have_no_license = False for file_license in file_licenses: spdx_id = file_license.get('spdx_license_key') if spdx_id: spdx_license = License.from_identifier(spdx_id) else: license_key = file_license.get('key') # FIXME: we should prefix this with ScanCode- licenseref_id = 'LicenseRef-' + license_key spdx_license = ExtractedLicense(licenseref_id) spdx_license.name = file_license.get('short_name') comment = 'See details at https://github.com/nexB/scancode-toolkit/blob/develop/src/licensedcode/data/licenses/%s.yml\n' % license_key spdx_license.comment = comment text = file_license.get('matched_text') # always set some text, even if we did not extract the matched text if not text: text = comment spdx_license.text = text doc.add_extr_lic(spdx_license) # Add licenses in the order they appear in the file. Maintaining the order # might be useful for provenance purposes. file_entry.add_lics(spdx_license) doc.package.add_lics_from_file(spdx_license) elif file_licenses is None: all_files_have_no_license = False file_entry.add_lics(NoAssert()) else: file_entry.add_lics(SPDXNone()) file_entry.conc_lics = NoAssert() file_copyrights = file_data.get('copyrights') if file_copyrights: all_files_have_no_copyright = False file_entry.copyright = [] for file_copyright in file_copyrights: file_entry.copyright.extend(file_copyright.get('statements')) doc.package.cr_text.update(file_entry.copyright) # Create a text of copyright statements in the order they appear in the file. # Maintaining the order might be useful for provenance purposes. file_entry.copyright = '\n'.join(file_entry.copyright) + '\n' elif file_copyrights is None: all_files_have_no_copyright = False file_entry.copyright = NoAssert() else: file_entry.copyright = SPDXNone() doc.package.add_file(file_entry) if len(doc.package.files) == 0: if format == 'spdx-tv': output_file.write("# No results for package '{}'.\n".format( doc.package.name)) elif format == 'spdx-rdf': output_file.write("<!-- No results for package '{}'. -->\n".format( doc.package.name)) # Remove duplicate licenses from the list for the package. unique_licenses = set(doc.package.licenses_from_files) if not len(doc.package.licenses_from_files): if all_files_have_no_license: doc.package.licenses_from_files = [SPDXNone()] else: doc.package.licenses_from_files = [NoAssert()] else: # List license identifiers alphabetically for the package. doc.package.licenses_from_files = sorted(unique_licenses, key=lambda x: x.identifier) if len(doc.package.cr_text) == 0: if all_files_have_no_copyright: doc.package.cr_text = SPDXNone() else: doc.package.cr_text = NoAssert() else: # Create a text of alphabetically sorted copyright # statements for the package. doc.package.cr_text = '\n'.join(sorted(doc.package.cr_text)) + '\n' doc.package.verif_code = doc.package.calc_verif_code() doc.package.license_declared = NoAssert() doc.package.conc_lics = NoAssert() if format == 'spdx-tv': from spdx.writers.tagvalue import write_document elif format == 'spdx-rdf': from spdx.writers.rdf import write_document # As the spdx-tools package can only write the document to a # "str" file but ScanCode provides a "unicode" file, write to a # "str" buffer first and then manually write the value to a # "unicode" file. from StringIO import StringIO str_buffer = StringIO() write_document(doc, str_buffer, validate=True) as_unicode = str_buffer.getvalue().decode('utf-8') output_file.write(as_unicode)