def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Query GraphDB,
graph_response = GraphAnalyses.get_batch_ca_data(
ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW"
) != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def test_add_unknown_pkg_info(self):
"""Test Known Unknown Pkgs, with and Without CVE for golang."""
input_pkgs = [
('github.com/hashicorp/nomad', 'github.com/hashicorp/nomad',
'0.7.1', 'v0.7.1', False),
('code.cloudfoundry.org/gorouter/route',
'code.cloudfoundry.org/gorouter/[email protected]/gorouter',
'0.0.0-20170410000936-a663fba25f7a',
'v0.0.0-20170410000936-a663fba25f7a', True)
]
unknown_pkgs = set(
normlize_packages(pkg, gvn_pkg, vr, gvn_vr, isp)
for pkg, gvn_pkg, vr, gvn_vr, isp in input_pkgs)
stack_recommendation = [{
"package": "github.com/existing/package",
"version": "v3.4.0",
"package_unknown": False
}]
ideal_output = [{
"package": "github.com/existing/package",
"version": "v3.4.0",
"package_unknown": False
}, {
"package": "github.com/hashicorp/nomad",
"version": "v0.7.1",
"package_unknown": True
}, {
"package":
"code.cloudfoundry.org/gorouter/[email protected]/gorouter",
"version": "v0.0.0-20170410000936-a663fba25f7a",
"package_unknown": True
}]
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
self.assertListEqual(
sorted(stack_recommendation, key=itemgetter('package')),
sorted(ideal_output, key=itemgetter('package')))