def component_analyses_get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
input_json = {
"package_versions": [{
"package": package,
"version": version,
}]
}
try:
ca_validate_input(input_json, ecosystem)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
if analyses_result is not None:
return jsonify(analyses_result)
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = {"message": f"No data found for {ecosystem} package {package}/{version}"}
return jsonify(msg), 404
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def initiate_unknown_package_ingestion(self):
"""Ingestion of Unknown dependencies."""
ecosystem = self._normalized_packages.ecosystem
pkg_list = self.get_all_unknown_packages()
unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name,
version=pkg.version), pkg_list))
try:
unknown_package_flow(ecosystem, unknown_pkgs)
except Exception as e:
logger.error('Unknown ingestion failed with %s', e)
else:
logger.debug('Unknown ingestion executed for %s packages in %s ecosystem',
len(pkg_list), ecosystem)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(
ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
