def __enter__(self): self._versions = PypiVersionAPI() self.set_api(self.collect_packages())
def __enter__(self): self._api_response = self._fetch() self._versions = PypiVersionAPI() self.set_api(self.collect_packages())
class SafetyDbDataSource(DataSource): CONFIG_CLASS = SafetyDbConfiguration def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._api_response = self._fetch() validate_schema(self._api_response) def __enter__(self): self._versions = PypiVersionAPI() self.set_api(self.collect_packages()) @property def versions(self): # quick hack to make it patchable return self._versions def set_api(self, packages): asyncio.run(self._versions.load_api(packages)) def _fetch(self) -> Mapping[str, Any]: if self.create_etag(self.config.url): with urlopen(self.config.url) as response: return json.load(response) return [] def collect_packages(self): return {pkg for pkg in self._api_response} def updated_advisories(self) -> Set[Advisory]: advisories = [] for package_name in self._api_response: all_package_versions = self.versions.get(package_name) if len(all_package_versions) == 0: # PyPi does not have data about this package, we skip these continue for advisory in self._api_response[package_name]: impacted_purls, resolved_purls = categorize_versions( package_name, all_package_versions, advisory["specs"]) cve_ids = advisory.get("cve") or [""] # meaning if cve_ids is not [''] but either ['CVE-123'] or ['CVE-123, CVE-124'] if len(cve_ids[0]): cve_ids = [s.strip() for s in cve_ids.split(",")] reference = [Reference(reference_id=advisory["id"])] for cve_id in cve_ids: advisories.append( Advisory( cve_id=cve_id, summary=advisory["advisory"], vuln_references=reference, impacted_package_urls=impacted_purls, resolved_package_urls=resolved_purls, )) return self.batch_advisories(advisories) def create_etag(self, url): etag = requests.head(url).headers.get('ETag') if not etag: # Kind of inaccurate to return True since etag is # not created return True elif url in self.config.etags: if self.config.etags[url] == etag: return False self.config.etags[url] = etag return True
class SafetyDbDataSource(DataSource): CONFIG_CLASS = SafetyDbConfiguration def __enter__(self): self._api_response = self._fetch() self._versions = PypiVersionAPI() self.set_api(self.collect_packages()) @property def versions(self): # quick hack to make it patchable return self._versions def set_api(self, packages): asyncio.run(self._versions.load_api(packages)) def _fetch(self) -> Mapping[str, Any]: if self.create_etag(self.config.url): return requests.get(self.config.url).json() return [] def collect_packages(self): return {pkg for pkg in self._api_response} def updated_advisories(self) -> Set[Advisory]: for package_name in self._api_response: if package_name == "$meta" or package_name == "cumin": # This is the first entry in the data feed. It contains metadata of the feed. # Skip it. The 'cumin' entry is wrong continue try: validate_schema(self._api_response[package_name]) except Exception as e: logger.error(e) continue all_package_versions = self.versions.get(package_name) if not len(all_package_versions): # PyPi does not have data about this package, we skip these continue for advisory in self._api_response[package_name]: if advisory["cve"]: # Check on advisory["cve"] instead of using `get` because it can have null value cve_ids = re.findall(r"CVE-\d+-\d+", advisory["cve"]) else: continue impacted_purls, resolved_purls = categorize_versions( package_name, all_package_versions, advisory["specs"]) reference = [Reference(reference_id=advisory["id"])] advisories = [] for cve_id in cve_ids: advisories.append( Advisory( vulnerability_id=cve_id, summary=advisory["advisory"], references=reference, affected_packages=nearest_patched_package( impacted_purls, resolved_purls), )) yield advisories # FIXME: This is duplicate code. Use the the helper instead. def create_etag(self, url): etag = requests.head(url).headers.get("ETag") if not etag: # Kind of inaccurate to return True since etag is # not created return True elif url in self.config.etags: if self.config.etags[url] == etag: return False self.config.etags[url] = etag return True