def test_get_known_unknown_pkgs_with_and_without_cve_golang(self, _mock1):
"""Test Known Unknown Pkgs, with and Without CVE for golang."""
input_pkgs = [
('github.com/hashicorp/nomad', 'github.com/hashicorp/nomad',
'0.7.1', 'v0.7.1', False),
('code.cloudfoundry.org/gorouter/route',
'code.cloudfoundry.org/gorouter/[email protected]/gorouter',
'0.0.0-20170410000936-a663fba25f7a',
'v0.0.0-20170410000936-a663fba25f7a', True)
]
normalised_input_pkgs = [
normlize_packages(pkg, gvn_pkg, vr, gvn_vr, isp)
for pkg, gvn_pkg, vr, gvn_vr, isp in input_pkgs
]
batch_data_no_cve = os.path.join(
'/bayesian/tests/data/gremlin/batch_data_with_n_without_cve_golang.json'
)
with open(batch_data_no_cve) as f:
data_with_n_without_cve = json.load(f)
ideal_resp = os.path.join(
'/bayesian/tests/data/response/ca_batch_with_n_without_vul_golang.json'
)
with open(ideal_resp) as f:
ideal_output = json.load(f)
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
"golang", data_with_n_without_cve, normalised_input_pkgs)
self.assertListEqual(stack_recommendation, ideal_output)
self.assertSetEqual(unknown_pkgs, set())
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def test_get_known_unknown_pkgs_no_cve(self, _mock1, _mock2):
"""Test Known Unknown Pkgs, No Cve."""
normalised_input_pkgs = [('markdown2', "2.3.2")]
batch_data_no_cve = os.path.join('/bayesian/tests/data/gremlin/batch_data_no_cve.json')
with open(batch_data_no_cve) as f:
gremlin_batch_data_no_cve = json.load(f)
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
"pypi", gremlin_batch_data_no_cve, normalised_input_pkgs)
ideal_output = [{'package': 'markdown2',
'version': '2.3.2',
'package_unknown': False,
'recommendation': {}}]
self.assertListEqual(stack_recommendation, ideal_output)
self.assertSetEqual(unknown_pkgs, set(normalised_input_pkgs))
def test_get_known_unknown_pkgs_with_and_without_cve(self, _mock1, _mock2, _mock3):
"""Test Known Unknown Pkgs, with and Without CVE."""
normalised_input_pkgs = [('flask', "1.1.1"), ('django', "1.1.1")]
batch_data_no_cve = os.path.join(
'/bayesian/tests/data/gremlin/batch_data_with_n_without_cve.json')
with open(batch_data_no_cve) as f:
data_with_n_without_cve = json.load(f)
ideal_resp = os.path.join(
'/bayesian/tests/data/response/ca_batch_with_n_without_vul.json')
with open(ideal_resp) as f:
ideal_output = json.load(f)
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
"pypi", data_with_n_without_cve, normalised_input_pkgs)
self.assertListEqual(stack_recommendation, ideal_output)
self.assertSetEqual(unknown_pkgs, set(normalised_input_pkgs))
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Query GraphDB,
graph_response = GraphAnalyses.get_batch_ca_data(
ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW"
) != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def test_get_known_unknown_pkgs_with_and_without_cve(self, _mock1):
"""Test Known Unknown Pkgs, with and Without CVE."""
input_pkgs = [("flask", "flask", "1.1.1", "1.1.1"),
("django", "django", "1.1.1", "1.1.1")]
normalised_input_pkgs = [
normlize_packages(pkg, gvn_pkg, vr, gvn_vr, False)
for pkg, gvn_pkg, vr, gvn_vr in input_pkgs
]
batch_data_no_cve = os.path.join(
'/bayesian/tests/data/gremlin/batch_data_with_n_without_cve.json')
with open(batch_data_no_cve) as f:
data_with_n_without_cve = json.load(f)
ideal_resp = os.path.join(
'/bayesian/tests/data/response/ca_batch_with_n_without_vul.json')
with open(ideal_resp) as f:
ideal_output = json.load(f)
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
"pypi", data_with_n_without_cve, normalised_input_pkgs)
self.assertListEqual(stack_recommendation, ideal_output)
self.assertSetEqual(unknown_pkgs, set())
