def server_create_analysis(ecosystem,
package,
version,
user_profile,
api_flow=True,
force=False,
force_graph_sync=False):
"""Create bayesianApiFlow handling analyses for specified EPV.
:param ecosystem: ecosystem for which the flow should be run
:param package: package for which should be flow run
:param version: package version
:param force: force run flow even specified EPV exists
:param force_graph_sync: force synchronization to graph
:return: dispatcher ID handling flow
"""
# Bookkeeping first
component = MavenCoordinates.normalize_str(
package) if ecosystem == 'maven' else package
server_create_component_bookkeeping(ecosystem, component, version,
user_profile)
args = {
'ecosystem': ecosystem,
'name': component,
'version': version,
'force': force,
'force_graph_sync': force_graph_sync
}
if api_flow:
return server_run_flow('bayesianApiFlow', args)
else:
return server_run_flow('bayesianFlow', args)
def _create_analysis_arguments(ecosystem, name, version):
"""Create arguments for analysis."""
return {
'ecosystem': ecosystem,
'name': MavenCoordinates.normalize_str(name) if ecosystem == 'maven' else name,
'version': version
}
def get(ecosystem, package, version):
decoded = decode_token()
package = urllib.parse.unquote(package)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
package = case_sensitivity_transform(ecosystem, package)
result = get_analyses_from_graph(ecosystem, package, version)
if result is not None:
# Known component for Bayesian
server_create_component_bookkeeping(ecosystem, package, version, decoded)
return result
if os.environ.get("INVOKE_API_WORKERS", "") == "1":
# Enter the unknown path
server_create_analysis(ecosystem, package, version, user_profile=decoded,
api_flow=True, force=False, force_graph_sync=True)
msg = "Package {ecosystem}/{package}/{version} is unavailable. " \
"The package will be available shortly," \
" please retry after some time.".format(ecosystem=ecosystem, package=package,
version=version)
raise HTTPError(202, msg)
else:
server_create_analysis(ecosystem, package, version, user_profile=decoded,
api_flow=False, force=False, force_graph_sync=True)
msg = "No data found for {ecosystem} Package " \
"{package}/{version}".format(ecosystem=ecosystem,
package=package, version=version)
raise HTTPError(404, msg)
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]:
"""Validate CA Input."""
logger.debug('Validating ca input data.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
gh = GithubUtils()
packages_list = []
normalised_input_pkgs = []
for pkg in input_json.get('package_versions'):
pseudo_version = False
package = given_package = pkg.get("package")
clean_version = given_version = pkg.get("version")
if not all([package, given_version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(given_version, str)) or (not isinstance(package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(given_version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
if ecosystem == 'pypi':
package = package.lower()
if ecosystem == 'golang':
_, clean_version = GolangDependencyTreeGenerator.clean_version(given_version)
pseudo_version = gh.is_pseudo_version(clean_version)
# Strip module appended to the package name
package = package.split('@')[0]
packages_list.append(
{"name": package, "given_name": given_package,
"version": clean_version, "given_version": given_version,
"is_pseudo_version": pseudo_version})
normalised_input_pkgs.append(normlize_packages(package, given_package, clean_version,
given_version, pseudo_version))
return packages_list, normalised_input_pkgs
def _create_analysis_arguments(ecosystem, name, version):
"""Create arguments for analysis."""
return {
'ecosystem': ecosystem,
'name': MavenCoordinates.normalize_str(name) if Ecosystem.by_name(
StoragePool.get_connected_storage('BayesianPostgres').session,
ecosystem).is_backed_by(
EcosystemBackend.maven) else name,
'version': version
}
def post(ecosystem, package, version):
decoded = decode_token()
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
package = case_sensitivity_transform(ecosystem, package)
server_create_analysis(ecosystem, package, version,
user_profile=decoded or {}, api_flow=True, force=True,
force_graph_sync=False)
return {}, 202
def validate_input(input_json: Dict,
ecosystem: str) -> (List[Dict], List[Dict]):
"""Validate Request Body."""
logger.debug('Validate Request Body.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
packages_list = []
invalid_packages = []
for pkg in input_json.get('package_versions'):
package = pkg.get("package")
clean_version = pkg.get("version")
if not all([package, clean_version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(clean_version, str)) or (not isinstance(
package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(clean_version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
# if package is invalid, add it to list of invalid packages
except Exception:
invalid_packages.append({
"name": package,
"version": clean_version,
"vulnerabilities": []
})
continue
if ecosystem == 'pypi':
package = package.lower()
packages_list.append({"name": package, "version": clean_version})
return packages_list, invalid_packages
def get_latest_analysis_for(ecosystem, package, version):
"""Note: has to be called inside flask request context"""
try:
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
return rdb.session.query(Analysis).\
join(Version).join(Package).join(Ecosystem).\
filter(Ecosystem.name == ecosystem).\
filter(Package.name == package).\
filter(Version.identifier == version).\
order_by(Analysis.started_at.desc()).\
first()
except NoResultFound:
return None
def get_latest_analysis_for(ecosystem, package, version):
"""Note: has to be called inside flask request context."""
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
try:
return rdb.session.query(Analysis).\
join(Version).join(Package).join(Ecosystem).\
filter(Ecosystem.name == ecosystem).\
filter(Package.name == package).\
filter(Version.identifier == version).\
order_by(Analysis.started_at.desc()).\
first()
except SQLAlchemyError:
rdb.session.rollback()
raise
def get(ecosystem, package, version):
"""Handle the GET REST API call."""
decoded = decode_token()
package = urllib.parse.unquote(package)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
package = case_sensitivity_transform(ecosystem, package)
result = get_analyses_from_graph(ecosystem, package, version)
if result is not None:
return result
msg = "No data found for {ecosystem} package " \
"{package}/{version}".format(ecosystem=ecosystem,
package=package, version=version)
raise HTTPError(404, msg)
def ca_validate_input(input_json: Dict,
ecosystem: str) -> Tuple[List[Dict], List[Package]]:
"""Validate CA Input."""
logger.debug('Validating ca input data.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
packages_list = []
normalised_input_pkgs = []
for pkg in input_json.get('package_versions'):
package = pkg.get("package")
version = pkg.get("version")
if not all([package, version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(version, str)) or (not isinstance(package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
if ecosystem == 'pypi':
package = package.lower()
packages_list.append({"name": package, "version": version})
normalised_input_pkgs.append(normlize_packages(package, version))
return packages_list, normalised_input_pkgs
def get_analysis_count(self, ecosystem, package):
"""Get count of previously scheduled analyses for given ecosystem-package.
:param ecosystem: str, Ecosystem name
:param package: str, Package name
:return: analysis count
"""
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
try:
count = PostgresBase.session.query(PackageAnalysis).\
join(Package).join(Ecosystem).\
filter(Ecosystem.name == ecosystem).\
filter(Package.name == package).\
count()
except SQLAlchemyError:
PostgresBase.session.rollback()
raise
return count
def get_analysis_count(ecosystem, package, version):
"""Get count of previously scheduled analysis for given EPV triplet.
:param ecosystem: str, Ecosystem name
:param package: str, Package name
:param version: str, Package version
:return: analysis count
"""
if Ecosystem.by_name(PostgresBase.session, ecosystem).is_backed_by(EcosystemBackend.maven):
package = MavenCoordinates.normalize_str(package)
try:
count = PostgresBase.session.query(Analysis).\
join(Version).join(Package).join(Ecosystem).\
filter(Ecosystem.name == ecosystem).\
filter(Package.name == package).\
filter(Version.identifier == version).\
count()
except SQLAlchemyError:
PostgresBase.session.rollback()
raise
return count
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(
ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known:
- DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202
- DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition.
- INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch
Package details
- INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch
Package details
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
# Known component for Fabric8 Analytics
server_create_component_bookkeeping(ecosystem, package, version,
g.decoded_token)
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1":
msg = f"No data found for {ecosystem} package {package}/{version} " \
"ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled"
return response_template({'error': msg}, 202)
if os.environ.get("INVOKE_API_WORKERS", "") == "1":
# Trigger the unknown component ingestion.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=True,
force=False,
force_graph_sync=True)
msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \
"The package will be available shortly," \
" please retry after some time."
metrics_payload.update({
"status_code": 202,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return response_template({'error': msg}, 202)
# No data has been found and INVOKE_API_WORKERS flag is down.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=False,
force=False,
force_graph_sync=True)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
