def __enter__(self):
self._versions = PypiVersionAPI()
self.set_api(self.collect_packages())
def __enter__(self):
self._api_response = self._fetch()
self._versions = PypiVersionAPI()
self.set_api(self.collect_packages())
class SafetyDbDataSource(DataSource):
CONFIG_CLASS = SafetyDbConfiguration
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._api_response = self._fetch()
validate_schema(self._api_response)
def __enter__(self):
self._versions = PypiVersionAPI()
self.set_api(self.collect_packages())
@property
def versions(self): # quick hack to make it patchable
return self._versions
def set_api(self, packages):
asyncio.run(self._versions.load_api(packages))
def _fetch(self) -> Mapping[str, Any]:
if self.create_etag(self.config.url):
with urlopen(self.config.url) as response:
return json.load(response)
return []
def collect_packages(self):
return {pkg for pkg in self._api_response}
def updated_advisories(self) -> Set[Advisory]:
advisories = []
for package_name in self._api_response:
all_package_versions = self.versions.get(package_name)
if len(all_package_versions) == 0:
# PyPi does not have data about this package, we skip these
continue
for advisory in self._api_response[package_name]:
impacted_purls, resolved_purls = categorize_versions(
package_name, all_package_versions, advisory["specs"])
cve_ids = advisory.get("cve") or [""]
# meaning if cve_ids is not [''] but either ['CVE-123'] or ['CVE-123, CVE-124']
if len(cve_ids[0]):
cve_ids = [s.strip() for s in cve_ids.split(",")]
reference = [Reference(reference_id=advisory["id"])]
for cve_id in cve_ids:
advisories.append(
Advisory(
cve_id=cve_id,
summary=advisory["advisory"],
vuln_references=reference,
impacted_package_urls=impacted_purls,
resolved_package_urls=resolved_purls,
))
return self.batch_advisories(advisories)
def create_etag(self, url):
etag = requests.head(url).headers.get('ETag')
if not etag:
# Kind of inaccurate to return True since etag is
# not created
return True
elif url in self.config.etags:
if self.config.etags[url] == etag:
return False
self.config.etags[url] = etag
return True
class SafetyDbDataSource(DataSource):
CONFIG_CLASS = SafetyDbConfiguration
def __enter__(self):
self._api_response = self._fetch()
self._versions = PypiVersionAPI()
self.set_api(self.collect_packages())
@property
def versions(self): # quick hack to make it patchable
return self._versions
def set_api(self, packages):
asyncio.run(self._versions.load_api(packages))
def _fetch(self) -> Mapping[str, Any]:
if self.create_etag(self.config.url):
return requests.get(self.config.url).json()
return []
def collect_packages(self):
return {pkg for pkg in self._api_response}
def updated_advisories(self) -> Set[Advisory]:
for package_name in self._api_response:
if package_name == "$meta" or package_name == "cumin":
# This is the first entry in the data feed. It contains metadata of the feed.
# Skip it. The 'cumin' entry is wrong
continue
try:
validate_schema(self._api_response[package_name])
except Exception as e:
logger.error(e)
continue
all_package_versions = self.versions.get(package_name)
if not len(all_package_versions):
# PyPi does not have data about this package, we skip these
continue
for advisory in self._api_response[package_name]:
if advisory["cve"]:
# Check on advisory["cve"] instead of using `get` because it can have null value
cve_ids = re.findall(r"CVE-\d+-\d+", advisory["cve"])
else:
continue
impacted_purls, resolved_purls = categorize_versions(
package_name, all_package_versions, advisory["specs"])
reference = [Reference(reference_id=advisory["id"])]
advisories = []
for cve_id in cve_ids:
advisories.append(
Advisory(
vulnerability_id=cve_id,
summary=advisory["advisory"],
references=reference,
affected_packages=nearest_patched_package(
impacted_purls, resolved_purls),
))
yield advisories
# FIXME: This is duplicate code. Use the the helper instead.
def create_etag(self, url):
etag = requests.head(url).headers.get("ETag")
if not etag:
# Kind of inaccurate to return True since etag is
# not created
return True
elif url in self.config.etags:
if self.config.etags[url] == etag:
return False
self.config.etags[url] = etag
return True