def analysis_records_exist(self, analysis_document: dict) -> bool:
"""Check whether the given analysis document records exist in the graph database."""
loop = asyncio.get_event_loop()
query = self.g.E() \
.has('__label__', IsPartOf.__label__) \
.has('__type__', 'edge') \
.has('analysis_datetime', datetime_str2timestamp(analysis_document['metadata']['datetime'])) \
.has('analysis_document_id', analysis_document['metadata']['hostname']) \
.has('analyzer_name', analysis_document['metadata']['analyzer']) \
.has('analyzer_version', analysis_document['metadata']['analyzer_version'])\
.count().next()
return loop.run_until_complete(query) > 0
def solver_records_exist(self, solver_document: dict) -> bool:
"""Check whether the given solver document record exists in the graph database."""
loop = asyncio.get_event_loop()
query = self.g.V() \
.has('__label__', EcosystemSolver.__label__) \
.has('__type__', 'vertex') \
.has('solver_name', solver_document['metadata']['analyzer']) \
.has('solver_version', solver_document['metadata']['analyzer_version']) \
.outE() \
.has('__type__', 'edge') \
.has('__label__', Solved.__label__) \
.has('solver_document_id', solver_document['metadata']['hostname']) \
.has('solver_datetime', datetime_str2timestamp(solver_document['metadata']['datetime'])) \
.count().next()
return loop.run_until_complete(query) > 0
def _python_sync_analysis_result(
self, document_id: str, document: dict,
runtime_environment: RuntimeEnvironment) -> None:
"""Sync results of Python packages found in the given container image."""
# or [] should go to analyzer to be consistent
for python_package_info in document['result']['mercator'] or []:
if python_package_info['ecosystem'] == 'Python-RequirementsTXT':
# We don't want to sync found requirement.txt artifacts as
# they do not carry any valuable information for us.
continue
if 'result' not in python_package_info or 'error' in python_package_info[
'result']:
# Mercator was unable to process this - e.g. there was a
# setup.py that is not distutils setup.py
_LOGGER.info("Skipping error entry - %r", python_package_info)
continue
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=python_package_info['result']['name'].lower(),
package_version=python_package_info['result']['version'])
IsPartOf.from_properties(
source=python_package_version,
target=runtime_environment,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document_id,
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-exception
_LOGGER.exception(
f"Failed to sync Python package, error is not fatal: {python_package_info!r}"
)
def _rpm_sync_analysis_result(
self, document_id: str, document: dict,
runtime_environment: RuntimeEnvironment) -> None:
"""Sync results of RPMs found in the given container image."""
for rpm_package_info in document['result']['rpm-dependencies']:
try:
rpm_package_version = RPMPackageVersion.from_properties(
ecosystem='rpm',
package_name=rpm_package_info['name'],
package_version=rpm_package_info['version'],
release=rpm_package_info.get('release'),
epoch=rpm_package_info.get('epoch'),
arch=rpm_package_info.get('arch'),
src=rpm_package_info.get('src', False),
package_identifier=rpm_package_info.get(
'package_identifier', rpm_package_info['name']))
rpm_package_version.get_or_create(self.g)
rpm_package = Package.from_properties(
ecosystem=rpm_package_version.ecosystem,
package_name=rpm_package_version.package_name,
)
rpm_package.get_or_create(self.g)
HasVersion.from_properties(
source=rpm_package,
target=rpm_package_version).get_or_create(self.g)
IsPartOf.from_properties(
source=rpm_package_version,
target=runtime_environment,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document_id,
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync RPM package, error is not fatal: {rpm_package_info!r}"
)
continue
for dependency in rpm_package_info['dependencies']:
try:
rpm_requirement = RPMRequirement.from_properties(
rpm_requirement_name=dependency)
rpm_requirement.get_or_create(self.g)
Requires.from_properties(
source=rpm_package_version,
target=rpm_requirement,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document_id,
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync dependencies for "
f"RPM {rpm_package_version.to_dict()}: {dependency!r}")
def _deb_sync_analysis_result(
self, document_id: str, document: dict,
runtime_environment: RuntimeEnvironment) -> None:
"""Sync results of deb packages found in the given container image."""
for deb_package_info in document['result']['deb-dependencies']:
try:
deb_package_version = DebPackageVersion.from_properties(
ecosystem='deb',
package_name=deb_package_info['name'],
package_version=deb_package_info['version'],
arch=deb_package_info['arch'],
epoch=deb_package_info.get('epoch'))
deb_package_version.get_or_create(self.g)
deb_package = Package.from_properties(
ecosystem=deb_package_version.ecosystem,
package_name=deb_package_version.package_name)
deb_package.get_or_create(self.g)
HasVersion.from_properties(
source=deb_package,
target=deb_package_version).get_or_create(self.g)
IsPartOf.from_properties(
source=deb_package_version,
target=runtime_environment,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document_id,
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
# These three can be grouped with a zip, but that is not that readable...
for pre_depends in deb_package_info.get('pre-depends') or []:
package = Package.from_properties(
ecosystem='deb', package_name=pre_depends['name'])
package.get_or_create(self.g)
DebPreDepends.from_properties(
source=deb_package_version,
target=package,
version_range=pre_depends.get(
'version')).get_or_create(self.g)
for depends in deb_package_info.get('depends') or []:
package = Package.from_properties(
ecosystem='deb', package_name=depends['name'])
package.get_or_create(self.g)
DebDepends.from_properties(
source=deb_package_version,
target=package,
version_range=depends.get('version')).get_or_create(
self.g)
for replaces in deb_package_info.get('replaces') or []:
package = Package.from_properties(
ecosystem='deb', package_name=replaces['name'])
package.get_or_create(self.g)
DebReplaces.from_properties(
source=deb_package_version,
target=package,
version_range=replaces.get('version')).get_or_create(
self.g)
except Exception:
_LOGGER.exception(
"Failed to sync debian package, error is not fatal: %r",
deb_package_info)
def sync_solver_result(self, document: dict) -> None:
"""Sync the given solver result to the graph database."""
ecosystem_solver = EcosystemSolver.from_properties(
solver_name=document['metadata']['analyzer'],
solver_version=document['metadata']['analyzer_version'])
ecosystem_solver.get_or_create(self.g)
solver_document_id = SolverResultsStore.get_document_id(document)
solver_datetime = datetime_str2timestamp(
document['metadata']['datetime'])
for python_package_info in document['result']['tree']:
try:
python_package, _, python_package_version = self.create_pypi_package_version(
python_package_info['package_name'].lower(),
python_package_info['package_version'])
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=False,
solver_error_unsolvable=False,
solver_error_unparsable=False).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync Python package, error is not fatal: {python_package_info!r}"
)
continue
for dependency in python_package_info['dependencies']:
try:
for dependency_version in dependency['resolved_versions']:
python_package_dependency, _, python_package_version_dependency = \
self.create_pypi_package_version(
package_name=dependency['package_name'],
package_version=dependency_version
)
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version_dependency,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=False,
solver_error_unsolvable=False,
solver_error_unparsable=False).get_or_create(
self.g)
# TODO: mark extras
DependsOn.from_properties(
source=python_package_version,
target=python_package_version_dependency,
package_name=python_package_version_dependency.
package_name.value,
version_range=dependency['required_version']
or '*').get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync Python package {python_package_version.to_dict()}"
f"dependency: {dependency}")
for error_info in document['result']['errors']:
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=error_info.get('package_name')
or error_info['package'],
package_version=error_info['version'],
)
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=True,
solver_error_unsolvable=False,
solver_error_unparsable=False).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
"Failed to sync Python package, error is not fatal: %r",
error_info)
for unsolvable in document['result']['unresolved']:
if not unsolvable['version_spec'].startswith('=='):
# No resolution can be perfomed so no identifier is captured, report warning and continue.
# We would like to capture this especially when there are
# packages in ecosystem that we cannot find (e.g. not configured private index
# or removed package).
_LOGGER.warning(
f"Cannot sync unsolvable package {unsolvable} as package is not locked to as specific version"
)
continue
package_version = unsolvable['version_spec'][len('=='):]
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=unsolvable['package_name'],
package_version=package_version,
)
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=True,
solver_error_unsolvable=True,
solver_error_unparsable=False).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
"Failed to sync unsolvable Python package, error is not fatal: %r",
unsolvable)
for unparsed in document['result']['unparsed']:
parts = unparsed.rsplit('==', maxsplit=1)
if len(parts) != 2:
# This request did not come from graph-refresh job as there is not pinned version.
_LOGGER.warning(
f"Cannot sync unparsed package {unparsed} as package is not locked to as specific version"
)
continue
package_name, package_version = parts
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=package_name,
package_version=package_version,
)
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=True,
solver_error_unsolvable=False,
solver_error_unparsable=True).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
"Failed to sync unparsed Python package, error is not fatal: %r",
unparsed)
def sync_analysis_result(self, document: dict) -> None:
"""Sync the given analysis result to the graph database."""
runtime_environment = RuntimeEnvironment.from_properties(
runtime_environment_name=document['metadata']['arguments']
['extract-image']['image'], )
runtime_environment.get_or_create(self.g)
# RPM packages
for rpm_package_info in document['result']['rpm-dependencies']:
try:
rpm_package_version = RPMPackageVersion.from_properties(
ecosystem='rpm',
package_name=rpm_package_info['name'],
package_version=rpm_package_info['version'],
release=rpm_package_info.get('release'),
epoch=rpm_package_info.get('epoch'),
arch=rpm_package_info.get('arch'),
src=rpm_package_info.get('src', False),
package_identifier=rpm_package_info.get(
'package_identifier', rpm_package_info['name']))
rpm_package_version.get_or_create(self.g)
rpm_package = Package.from_properties(
ecosystem=rpm_package_version.ecosystem,
package_name=rpm_package_version.package_name,
)
rpm_package.get_or_create(self.g)
HasVersion.from_properties(
source=rpm_package,
target=rpm_package_version).get_or_create(self.g)
IsPartOf.from_properties(
source=rpm_package_version,
target=runtime_environment,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document['metadata']['hostname'],
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync RPM package, error is not fatal: {rpm_package_info!r}"
)
continue
for dependency in rpm_package_info['dependencies']:
try:
rpm_requirement = RPMRequirement.from_properties(
rpm_requirement_name=dependency)
rpm_requirement.get_or_create(self.g)
Requires.from_properties(
source=rpm_package_version,
target=rpm_requirement,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document['metadata']['hostname'],
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync dependencies for "
f"RPM {rpm_package_version.to_dict()}: {dependency!r}")
# Python packages
for python_package_info in document['result'][
'mercator'] or []: # or [] should go to analyzer to be consistent
if python_package_info['ecosystem'] == 'Python-RequirementsTXT':
# We don't want to sync found requirement.txt artifacts as they do not carry any
# valuable information for us.
continue
if 'result' not in python_package_info or 'error' in python_package_info[
'result']:
# Mercator was unable to process this - e.g. there was a setup.py that is not distutils setup.py
_LOGGER.info("Skipping error entry - %r", python_package_info)
continue
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=python_package_info['result']['name'].lower(),
package_version=python_package_info['result']['version'])
IsPartOf.from_properties(
source=python_package_version,
target=runtime_environment,
analysis_datetime=datetime_str2timestamp(
document['metadata']['datetime']),
analysis_document_id=document['metadata']['hostname'],
analyzer_name=document['metadata']['analyzer'],
analyzer_version=document['metadata']
['analyzer_version']).get_or_create(self.g)
except Exception: # pylint: disable=broad-exception
_LOGGER.exception(
f"Failed to sync Python package, error is not fatal: {python_package_info!r}"
)
def sync_solver_result(self, document: dict) -> None:
"""Sync the given solver result to the graph database."""
ecosystem_solver = EcosystemSolver.from_properties(
solver_name=document['metadata']['analyzer'],
solver_version=document['metadata']['analyzer_version'])
ecosystem_solver.get_or_create(self.g)
solver_document_id = document['metadata']['hostname']
solver_datetime = datetime_str2timestamp(
document['metadata']['datetime'])
for python_package_info in document['result']['tree']:
try:
python_package, _, python_package_version = self.create_pypi_package_version(
python_package_info['package_name'].lower(),
python_package_info['package_version'])
Solved.from_properties(source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=False).get_or_create(
self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync Python package, error is not fatal: {python_package_info!r}"
)
continue
for dependency in python_package_info['dependencies']:
try:
for dependency_version in dependency['resolved_versions']:
python_package_dependency, _, python_package_version_dependency = \
self.create_pypi_package_version(
package_name=dependency['package_name'],
package_version=dependency_version
)
Solved.from_properties(
source=ecosystem_solver,
target=python_package_version_dependency,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=False).get_or_create(self.g)
# TODO: mark extras
DependsOn.from_properties(
source=python_package_version,
target=python_package_version_dependency,
package_name=python_package_version_dependency.
package_name.value,
version_range=dependency['required_version']
or '*').get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync Python package {python_package_version.to_dict()} "
f"dependency: {dependency}")
for error_info in document['result']['errors']:
try:
python_package, _, python_package_version = self.create_pypi_package_version(
package_name=error_info.get('package_name')
or error_info['package'],
package_version=error_info['version'],
)
Solved.from_properties(source=ecosystem_solver,
target=python_package_version,
solver_document_id=solver_document_id,
solver_datetime=solver_datetime,
solver_error=True).get_or_create(self.g)
except Exception: # pylint: disable=broad-except
_LOGGER.exception(
f"Failed to sync Python package, error is not fatal: {error_info!r}"
)