def test_check_for_accepted_ecosystem(): """Test check_for_accepted_ecosystem function.""" resp = check_for_accepted_ecosystem("maven") assert resp resp = check_for_accepted_ecosystem("abcd") assert not resp
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]: """Validate CA Input.""" logger.debug('Validating ca input data.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) gh = GithubUtils() packages_list = [] normalised_input_pkgs = [] for pkg in input_json.get('package_versions'): pseudo_version = False package = given_package = pkg.get("package") clean_version = given_version = pkg.get("version") if not all([package, given_version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(given_version, str)) or (not isinstance(package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(given_version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) if ecosystem == 'pypi': package = package.lower() if ecosystem == 'golang': _, clean_version = GolangDependencyTreeGenerator.clean_version(given_version) pseudo_version = gh.is_pseudo_version(clean_version) # Strip module appended to the package name package = package.split('@')[0] packages_list.append( {"name": package, "given_name": given_package, "version": clean_version, "given_version": given_version, "is_pseudo_version": pseudo_version}) normalised_input_pkgs.append(normlize_packages(package, given_package, clean_version, given_version, pseudo_version)) return packages_list, normalised_input_pkgs
def validate_input(input_json: Dict, ecosystem: str) -> (List[Dict], List[Dict]): """Validate Request Body.""" logger.debug('Validate Request Body.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) packages_list = [] invalid_packages = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") clean_version = pkg.get("version") if not all([package, clean_version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(clean_version, str)) or (not isinstance( package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(clean_version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) # if package is invalid, add it to list of invalid packages except Exception: invalid_packages.append({ "name": package, "version": clean_version, "vulnerabilities": [] }) continue if ecosystem == 'pypi': package = package.lower() packages_list.append({"name": package, "version": clean_version}) return packages_list, invalid_packages
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]: """Validate CA Input.""" logger.debug('Validating ca input data.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) packages_list = [] normalised_input_pkgs = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") version = pkg.get("version") if not all([package, version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(version, str)) or (not isinstance(package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) if ecosystem == 'pypi': package = package.lower() packages_list.append({"name": package, "version": version}) normalised_input_pkgs.append(normlize_packages(package, version)) return packages_list, normalised_input_pkgs
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result # No data has been found unknown_pkgs = set() unknown_pkgs.add( ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: - DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202 - DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition. - INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch Package details - INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch Package details :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: # Known component for Fabric8 Analytics server_create_component_bookkeeping(ecosystem, package, version, g.decoded_token) metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1": msg = f"No data found for {ecosystem} package {package}/{version} " \ "ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled" return response_template({'error': msg}, 202) if os.environ.get("INVOKE_API_WORKERS", "") == "1": # Trigger the unknown component ingestion. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=True, force=False, force_graph_sync=True) msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \ "The package will be available shortly," \ " please retry after some time." metrics_payload.update({ "status_code": 202, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return response_template({'error': msg}, 202) # No data has been found and INVOKE_API_WORKERS flag is down. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=False, force=False, force_graph_sync=True) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)