def fetch_licenses(self, scancode_licenses, from_repo=SPDX_DEFAULT_REPO): """ Yield License objects fetched from the latest SPDX license list. """ # get latest tag tags_url = 'https://api.github.com/repos/{from_repo}/tags'.format( **locals()) tags = get_response(tags_url, headers={}, params={}) tag = tags[0]['name'] # fetch licenses and exceptions # note that exceptions data have -- weirdly enough -- a different schema zip_url = 'https://github.com/{from_repo}/archive/{tag}.zip'.format( **locals()) if TRACE_FETCH: print('Fetching SPDX license data version:', tag, 'from:', zip_url) licenses_zip = fetch.download_url(zip_url, timeout=120) if TRACE_FETCH: print('Fteched SPDX licenses to:', licenses_zip) with zipfile.ZipFile(licenses_zip) as archive: for path in archive.namelist(): if not (path.endswith('.json') and ('/json/details/' in path or '/json/exceptions/' in path)): continue if TRACE_FETCH: print('Loading license:', path) if path.endswith('+.json'): # Skip the old plus licenses. We use them in # ScanCode, but they are deprecated in SPDX. continue details = json.loads(archive.read(path)) lic = self.build_license(details, scancode_licenses) if lic: yield lic
def fetch_licenses(self): """ Yield all the latest License object from the latest SPDX license list. Store the texts in the license_dir. """ # get latest tag tags_url = 'https://api.github.com/repos/spdx/license-list-data/tags' tags = get_response(tags_url, headers={}, params={}) tag = tags[0]['name'] # fetch licenses and exceptions # note that exceptions data have -- weirdly enough -- a different schema zip_url = 'https://github.com/spdx/license-list-data/archive/%(tag)s.zip' % locals() if TRACE_FETCH: print('Fetching SPDX license data from:', zip_url) licenses_zip = fetch.download_url(zip_url, timeout=120) with zipfile.ZipFile(licenses_zip) as archive: for path in archive.namelist(): if not (path.endswith('.json') and ('/json/details/' in path or '/json/exceptions/' in path)): continue if TRACE_FETCH: print('Loading license:', path) if path.endswith('+.json'): # Skip the old plus licenses. We use them in # ScanCode, but they are deprecated in SPDX. continue details = json.loads(archive.read(path)) lic = self._build_license(details) if lic: yield lic
def fetch_licenses(self): """ Yield all the latest License object from the latest SPDX license list. Store the texts in the license_dir. """ # get latest tag tags_url = 'https://api.github.com/repos/spdx/license-list-data/tags' tags = get_response(tags_url, headers={}, params={}) tag = tags[0]['name'] # fetch licenses and exceptions # note that exceptions data have -- weirdly enough -- a different schema zip_url = 'https://github.com/spdx/license-list-data/archive/%(tag)s.zip' % locals( ) if TRACE_FETCH: print('Fetching SPDX license data from:', zip_url) licenses_zip = fetch.download_url(zip_url, timeout=120) with zipfile.ZipFile(licenses_zip) as archive: for path in archive.namelist(): if not (path.endswith('.json') and ('/json/details/' in path or '/json/exceptions/' in path)): continue if TRACE_FETCH: print('Loading license:', path) if path.endswith('+.json'): # Skip the old plus licenses. We use them in # ScanCode, but they are deprecated in SPDX. continue details = json.loads(archive.read(path)) lic = self._build_license(details) if lic: yield lic
def fetch_text(url): """ Fetch and return a temp file from the content at `url`. """ if 'raw' in url: fetchable = url elif 'github.com' in url and '/blob/' in url: fetchable = url.replace('/blob/', '/raw/') else: fetchable = url print(' Fetching:', fetchable) return fetch.download_url(fetchable, timeout=120)
def fetch_text(url): """ Fetch and return a temp file from the content at `url`. """ if "raw" in url: fetchable = url elif "github.com" in url and "/blob/" in url: fetchable = url.replace("/blob/", "/raw/") else: fetchable = url print(" Fetching:", fetchable) return fetch.download_url(fetchable, timeout=120)
def fetch_licenses( self, scancode_licenses=None, commitish=None, skip_oddities=True, from_repo=SPDX_DEFAULT_REPO, ): """ Yield License objects fetched from the latest SPDX license list. Use the latest tagged version or the `commitish` if provided. If skip_oddities is True, some oddities are skipped or handled specially, such as licenses with a trailing + or foreign language licenses. """ if not commitish: # get latest tag tags_url = 'https://api.github.com/repos/{from_repo}/tags'.format( **locals()) tags = get_response(tags_url, headers={}, params={}) tag = tags[0]['name'] else: tag = commitish # fetch licenses and exceptions # note that exceptions data have -- weirdly enough -- a different schema zip_url = 'https://github.com/{from_repo}/archive/{tag}.zip'.format( **locals()) if TRACE_FETCH: print('Fetching SPDX license data version:', tag, 'from:', zip_url) licenses_zip = fetch.download_url(zip_url, timeout=120) if TRACE_FETCH: print('Fetched SPDX licenses to:', licenses_zip) with zipfile.ZipFile(licenses_zip) as archive: for path in archive.namelist(): if not (path.endswith('.json') and ('/json/details/' in path or '/json/exceptions/' in path)): continue if TRACE_FETCH: print('Loading license:', path) if skip_oddities and path.endswith('+.json'): # Skip the old plus licenses. We use them in # ScanCode, but they are deprecated in SPDX. continue details = json.loads(archive.read(path)) lic = self.build_license( mapping=details, scancode_licenses=scancode_licenses, skip_oddities=skip_oddities, ) if lic: yield lic