def component_analyses_get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
input_json = {
"package_versions": [{
"package": package,
"version": version,
}]
}
try:
ca_validate_input(input_json, ecosystem)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
if analyses_result is not None:
return jsonify(analyses_result)
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = {"message": f"No data found for {ecosystem} package {package}/{version}"}
return jsonify(msg), 404
def test_ca_validate_input_golang(self):
"""Test Ca Validate input: Ecosystem golang."""
input_json = {
"ecosystem": "golang",
"package_versions": [
{"package": "github.com/cmp/[email protected]/cmp", "version": "v1.2.4"},
{"package": "github.com/cmp/[email protected]/cmp",
"version": "v1.2.4+incompatible"},
{"package": "github.com/str/cmp", "version": "v0.0.0-20201010080808-abcd1234abcd"},
{"package": "github.com/str/extract", "version": "v1.0.5-alpha1.5"},
{"package": "github.com/str/merge", "version": "v3.4.5-alpha1.2+incompatible"},
]
}
ideal_result = [
{"name": "github.com/cmp/cmp-opt",
"given_name": "github.com/cmp/[email protected]/cmp", "version": "1.2.4",
"given_version": "v1.2.4", "is_pseudo_version": False},
{"name": "github.com/cmp/version",
"given_name": "github.com/cmp/[email protected]/cmp", "version": "1.2.4",
"given_version": "v1.2.4+incompatible", "is_pseudo_version": False},
{"name": "github.com/str/cmp", "given_name": "github.com/str/cmp",
"version": "0.0.0-20201010080808-abcd1234abcd",
"given_version": "v0.0.0-20201010080808-abcd1234abcd", "is_pseudo_version": True},
{"name": "github.com/str/extract", "given_name": "github.com/str/extract",
"version": "1.0.5-alpha1.5",
"given_version": "v1.0.5-alpha1.5", "is_pseudo_version": False},
{"name": "github.com/str/merge", "given_name": "github.com/str/merge",
"version": "3.4.5-alpha1.2",
"given_version": "v3.4.5-alpha1.2+incompatible", "is_pseudo_version": False}
]
result, _ = ca_validate_input(input_json, input_json["ecosystem"])
self.assertEqual(result, ideal_result)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def test_ca_validate_input_maven(self):
"""Test Ca Validate input: Ecosystem maven."""
input_json = {
"ecosystem": "maven",
"package_versions": [
{"package": "com.thoughtworks.xstream:xstream", "version": "1.3"},
]
}
ideal_result = [{"name": "com.thoughtworks.xstream:xstream", "version": "1.3"}]
result, _ = ca_validate_input(input_json, input_json["ecosystem"])
self.assertEqual(result, ideal_result)
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Query GraphDB,
graph_response = GraphAnalyses.get_batch_ca_data(
ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW"
) != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
