def test_check_for_accepted_ecosystem():
"""Test check_for_accepted_ecosystem function."""
resp = check_for_accepted_ecosystem("maven")
assert resp
resp = check_for_accepted_ecosystem("abcd")
assert not resp
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]:
"""Validate CA Input."""
logger.debug('Validating ca input data.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
gh = GithubUtils()
packages_list = []
normalised_input_pkgs = []
for pkg in input_json.get('package_versions'):
pseudo_version = False
package = given_package = pkg.get("package")
clean_version = given_version = pkg.get("version")
if not all([package, given_version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(given_version, str)) or (not isinstance(package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(given_version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
if ecosystem == 'pypi':
package = package.lower()
if ecosystem == 'golang':
_, clean_version = GolangDependencyTreeGenerator.clean_version(given_version)
pseudo_version = gh.is_pseudo_version(clean_version)
# Strip module appended to the package name
package = package.split('@')[0]
packages_list.append(
{"name": package, "given_name": given_package,
"version": clean_version, "given_version": given_version,
"is_pseudo_version": pseudo_version})
normalised_input_pkgs.append(normlize_packages(package, given_package, clean_version,
given_version, pseudo_version))
return packages_list, normalised_input_pkgs
def validate_input(input_json: Dict,
ecosystem: str) -> (List[Dict], List[Dict]):
"""Validate Request Body."""
logger.debug('Validate Request Body.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
packages_list = []
invalid_packages = []
for pkg in input_json.get('package_versions'):
package = pkg.get("package")
clean_version = pkg.get("version")
if not all([package, clean_version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(clean_version, str)) or (not isinstance(
package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(clean_version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
# if package is invalid, add it to list of invalid packages
except Exception:
invalid_packages.append({
"name": package,
"version": clean_version,
"vulnerabilities": []
})
continue
if ecosystem == 'pypi':
package = package.lower()
packages_list.append({"name": package, "version": clean_version})
return packages_list, invalid_packages
def ca_validate_input(input_json: Dict,
ecosystem: str) -> Tuple[List[Dict], List[Package]]:
"""Validate CA Input."""
logger.debug('Validating ca input data.')
if not input_json:
error_msg = "Expected JSON request"
raise BadRequest(error_msg)
if not isinstance(input_json, dict):
error_msg = "Expected list of dependencies in JSON request"
raise BadRequest(error_msg)
if not check_for_accepted_ecosystem(ecosystem):
error_msg: str = f"Ecosystem {ecosystem} is not supported for this request"
raise BadRequest(error_msg)
if not input_json.get('package_versions'):
error_msg: str = "package_versions is missing"
raise BadRequest(error_msg)
packages_list = []
normalised_input_pkgs = []
for pkg in input_json.get('package_versions'):
package = pkg.get("package")
version = pkg.get("version")
if not all([package, version]):
error_msg = "Invalid Input: Package, Version are required."
raise BadRequest(error_msg)
if (not isinstance(version, str)) or (not isinstance(package, str)):
error_msg = "Package version should be string format only."
raise BadRequest(error_msg)
if not validate_version(version):
error_msg = "Package version should not have special characters."
raise BadRequest(error_msg)
if ecosystem == 'maven':
package = MavenCoordinates.normalize_str(package)
if ecosystem == 'pypi':
package = package.lower()
packages_list.append({"name": package, "version": version})
normalised_input_pkgs.append(normlize_packages(package, version))
return packages_list, normalised_input_pkgs
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(
ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known:
- DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202
- DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition.
- INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch
Package details
- INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch
Package details
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
# Known component for Fabric8 Analytics
server_create_component_bookkeeping(ecosystem, package, version,
g.decoded_token)
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1":
msg = f"No data found for {ecosystem} package {package}/{version} " \
"ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled"
return response_template({'error': msg}, 202)
if os.environ.get("INVOKE_API_WORKERS", "") == "1":
# Trigger the unknown component ingestion.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=True,
force=False,
force_graph_sync=True)
msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \
"The package will be available shortly," \
" please retry after some time."
metrics_payload.update({
"status_code": 202,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return response_template({'error': msg}, 202)
# No data has been found and INVOKE_API_WORKERS flag is down.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=False,
force=False,
force_graph_sync=True)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
