def test_get_known_unknown_pkgs_with_and_without_cve_golang(self, _mock1): """Test Known Unknown Pkgs, with and Without CVE for golang.""" input_pkgs = [ ('github.com/hashicorp/nomad', 'github.com/hashicorp/nomad', '0.7.1', 'v0.7.1', False), ('code.cloudfoundry.org/gorouter/route', 'code.cloudfoundry.org/gorouter/[email protected]/gorouter', '0.0.0-20170410000936-a663fba25f7a', 'v0.0.0-20170410000936-a663fba25f7a', True) ] normalised_input_pkgs = [ normlize_packages(pkg, gvn_pkg, vr, gvn_vr, isp) for pkg, gvn_pkg, vr, gvn_vr, isp in input_pkgs ] batch_data_no_cve = os.path.join( '/bayesian/tests/data/gremlin/batch_data_with_n_without_cve_golang.json' ) with open(batch_data_no_cve) as f: data_with_n_without_cve = json.load(f) ideal_resp = os.path.join( '/bayesian/tests/data/response/ca_batch_with_n_without_vul_golang.json' ) with open(ideal_resp) as f: ideal_output = json.load(f) stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( "golang", data_with_n_without_cve, normalised_input_pkgs) self.assertListEqual(stack_recommendation, ideal_output) self.assertSetEqual(unknown_pkgs, set())
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') if request.user_agent.string == "claircore/crda/RemoteMatcher": try: md5_hash = hashlib.md5( json.dumps(input_json, sort_keys=True).encode('utf-8')).hexdigest() logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem, md5_hash) except Exception as e: logger.error("Exception %s", e) return jsonify({"message": "disabled"}), 404 try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs, input_json.get("ignore", {})) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"]) input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') user_agent = request.headers.get('User-Agent', None) manifest_hash = str(request.headers.get('manifest_hash', None)) request_id = request.headers.get('request_id', None) headers = {"uuid": request.headers.get('uuid', None)} try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, headers.get("uuid"), user_agent, manifest_hash, request_id) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info( stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return response_template(stack_recommendation, 202, headers) return response_template(stack_recommendation, 200, headers)
def test_get_known_unknown_pkgs_no_cve(self, _mock1, _mock2): """Test Known Unknown Pkgs, No Cve.""" normalised_input_pkgs = [('markdown2', "2.3.2")] batch_data_no_cve = os.path.join('/bayesian/tests/data/gremlin/batch_data_no_cve.json') with open(batch_data_no_cve) as f: gremlin_batch_data_no_cve = json.load(f) stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( "pypi", gremlin_batch_data_no_cve, normalised_input_pkgs) ideal_output = [{'package': 'markdown2', 'version': '2.3.2', 'package_unknown': False, 'recommendation': {}}] self.assertListEqual(stack_recommendation, ideal_output) self.assertSetEqual(unknown_pkgs, set(normalised_input_pkgs))
def test_get_known_unknown_pkgs_with_and_without_cve(self, _mock1, _mock2, _mock3): """Test Known Unknown Pkgs, with and Without CVE.""" normalised_input_pkgs = [('flask', "1.1.1"), ('django', "1.1.1")] batch_data_no_cve = os.path.join( '/bayesian/tests/data/gremlin/batch_data_with_n_without_cve.json') with open(batch_data_no_cve) as f: data_with_n_without_cve = json.load(f) ideal_resp = os.path.join( '/bayesian/tests/data/response/ca_batch_with_n_without_vul.json') with open(ideal_resp) as f: ideal_output = json.load(f) stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( "pypi", data_with_n_without_cve, normalised_input_pkgs) self.assertListEqual(stack_recommendation, ideal_output) self.assertSetEqual(unknown_pkgs, set(normalised_input_pkgs))
def post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"]) input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') headers = {"uuid": request.headers.get('uuid', None)} try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Query GraphDB, graph_response = GraphAnalyses.get_batch_ca_data( ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info( stack_recommendation, unknown_pkgs) if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW" ) != "1" and ecosystem != "golang": # Unknown Packages is Present and INGESTION is Enabled logger.debug(unknown_pkgs) unknown_package_flow(ecosystem, unknown_pkgs) return response_template(stack_recommendation, 202, headers) return response_template(stack_recommendation, 200, headers)
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def test_get_known_unknown_pkgs_with_and_without_cve(self, _mock1): """Test Known Unknown Pkgs, with and Without CVE.""" input_pkgs = [("flask", "flask", "1.1.1", "1.1.1"), ("django", "django", "1.1.1", "1.1.1")] normalised_input_pkgs = [ normlize_packages(pkg, gvn_pkg, vr, gvn_vr, False) for pkg, gvn_pkg, vr, gvn_vr in input_pkgs ] batch_data_no_cve = os.path.join( '/bayesian/tests/data/gremlin/batch_data_with_n_without_cve.json') with open(batch_data_no_cve) as f: data_with_n_without_cve = json.load(f) ideal_resp = os.path.join( '/bayesian/tests/data/response/ca_batch_with_n_without_vul.json') with open(ideal_resp) as f: ideal_output = json.load(f) stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( "pypi", data_with_n_without_cve, normalised_input_pkgs) self.assertListEqual(stack_recommendation, ideal_output) self.assertSetEqual(unknown_pkgs, set())