def component_analyses_get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ input_json = { "package_versions": [{ "package": package, "version": version, }] } try: ca_validate_input(input_json, ecosystem) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br if analyses_result is not None: return jsonify(analyses_result) # No data has been found unknown_pkgs = set() unknown_pkgs.add(ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = {"message": f"No data found for {ecosystem} package {package}/{version}"} return jsonify(msg), 404
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') if request.user_agent.string == "claircore/crda/RemoteMatcher": try: md5_hash = hashlib.md5( json.dumps(input_json, sort_keys=True).encode('utf-8')).hexdigest() logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem, md5_hash) except Exception as e: logger.error("Exception %s", e) return jsonify({"message": "disabled"}), 404 try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs, input_json.get("ignore", {})) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"]) input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') user_agent = request.headers.get('User-Agent', None) manifest_hash = str(request.headers.get('manifest_hash', None)) request_id = request.headers.get('request_id', None) headers = {"uuid": request.headers.get('uuid', None)} try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, headers.get("uuid"), user_agent, manifest_hash, request_id) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info( stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return response_template(stack_recommendation, 202, headers) return response_template(stack_recommendation, 200, headers)
def initiate_unknown_package_ingestion(self): """Ingestion of Unknown dependencies.""" ecosystem = self._normalized_packages.ecosystem pkg_list = self.get_all_unknown_packages() unknown_pkgs = set(map(lambda pkg: ingestion_utils.Package(package=pkg.name, version=pkg.version), pkg_list)) try: unknown_package_flow(ecosystem, unknown_pkgs) except Exception as e: logger.error('Unknown ingestion failed with %s', e) else: logger.debug('Unknown ingestion executed for %s packages in %s ecosystem', len(pkg_list), ecosystem)
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def test_ingest_epv_failed(self, _session): """Test unknown_package_flow negative.""" _session.side_effect = Exception(mock.Mock(return_value={'status': 404}), 'not found') with self.assertRaises(Exception): unknown_package_flow('dummy_eco', data_v1)
def test_ingest_epv(self, _session): """Test unknown_package_flow positive.""" unknown_package_flow('dummy_eco', data_v1)
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result # No data has been found unknown_pkgs = set() unknown_pkgs.add( ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)