def server_create_analysis(ecosystem, package, version, user_profile, api_flow=True, force=False, force_graph_sync=False): """Create bayesianApiFlow handling analyses for specified EPV. :param ecosystem: ecosystem for which the flow should be run :param package: package for which should be flow run :param version: package version :param force: force run flow even specified EPV exists :param force_graph_sync: force synchronization to graph :return: dispatcher ID handling flow """ # Bookkeeping first component = MavenCoordinates.normalize_str( package) if ecosystem == 'maven' else package server_create_component_bookkeeping(ecosystem, component, version, user_profile) args = { 'ecosystem': ecosystem, 'name': component, 'version': version, 'force': force, 'force_graph_sync': force_graph_sync } if api_flow: return server_run_flow('bayesianApiFlow', args) else: return server_run_flow('bayesianFlow', args)
def _create_analysis_arguments(ecosystem, name, version): """Create arguments for analysis.""" return { 'ecosystem': ecosystem, 'name': MavenCoordinates.normalize_str(name) if ecosystem == 'maven' else name, 'version': version }
def get(ecosystem, package, version): decoded = decode_token() package = urllib.parse.unquote(package) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) package = case_sensitivity_transform(ecosystem, package) result = get_analyses_from_graph(ecosystem, package, version) if result is not None: # Known component for Bayesian server_create_component_bookkeeping(ecosystem, package, version, decoded) return result if os.environ.get("INVOKE_API_WORKERS", "") == "1": # Enter the unknown path server_create_analysis(ecosystem, package, version, user_profile=decoded, api_flow=True, force=False, force_graph_sync=True) msg = "Package {ecosystem}/{package}/{version} is unavailable. " \ "The package will be available shortly," \ " please retry after some time.".format(ecosystem=ecosystem, package=package, version=version) raise HTTPError(202, msg) else: server_create_analysis(ecosystem, package, version, user_profile=decoded, api_flow=False, force=False, force_graph_sync=True) msg = "No data found for {ecosystem} Package " \ "{package}/{version}".format(ecosystem=ecosystem, package=package, version=version) raise HTTPError(404, msg)
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]: """Validate CA Input.""" logger.debug('Validating ca input data.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) gh = GithubUtils() packages_list = [] normalised_input_pkgs = [] for pkg in input_json.get('package_versions'): pseudo_version = False package = given_package = pkg.get("package") clean_version = given_version = pkg.get("version") if not all([package, given_version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(given_version, str)) or (not isinstance(package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(given_version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) if ecosystem == 'pypi': package = package.lower() if ecosystem == 'golang': _, clean_version = GolangDependencyTreeGenerator.clean_version(given_version) pseudo_version = gh.is_pseudo_version(clean_version) # Strip module appended to the package name package = package.split('@')[0] packages_list.append( {"name": package, "given_name": given_package, "version": clean_version, "given_version": given_version, "is_pseudo_version": pseudo_version}) normalised_input_pkgs.append(normlize_packages(package, given_package, clean_version, given_version, pseudo_version)) return packages_list, normalised_input_pkgs
def _create_analysis_arguments(ecosystem, name, version): """Create arguments for analysis.""" return { 'ecosystem': ecosystem, 'name': MavenCoordinates.normalize_str(name) if Ecosystem.by_name( StoragePool.get_connected_storage('BayesianPostgres').session, ecosystem).is_backed_by( EcosystemBackend.maven) else name, 'version': version }
def post(ecosystem, package, version): decoded = decode_token() if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) package = case_sensitivity_transform(ecosystem, package) server_create_analysis(ecosystem, package, version, user_profile=decoded or {}, api_flow=True, force=True, force_graph_sync=False) return {}, 202
def validate_input(input_json: Dict, ecosystem: str) -> (List[Dict], List[Dict]): """Validate Request Body.""" logger.debug('Validate Request Body.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) packages_list = [] invalid_packages = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") clean_version = pkg.get("version") if not all([package, clean_version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(clean_version, str)) or (not isinstance( package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(clean_version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) # if package is invalid, add it to list of invalid packages except Exception: invalid_packages.append({ "name": package, "version": clean_version, "vulnerabilities": [] }) continue if ecosystem == 'pypi': package = package.lower() packages_list.append({"name": package, "version": clean_version}) return packages_list, invalid_packages
def get_latest_analysis_for(ecosystem, package, version): """Note: has to be called inside flask request context""" try: if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) return rdb.session.query(Analysis).\ join(Version).join(Package).join(Ecosystem).\ filter(Ecosystem.name == ecosystem).\ filter(Package.name == package).\ filter(Version.identifier == version).\ order_by(Analysis.started_at.desc()).\ first() except NoResultFound: return None
def get_latest_analysis_for(ecosystem, package, version): """Note: has to be called inside flask request context.""" if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) try: return rdb.session.query(Analysis).\ join(Version).join(Package).join(Ecosystem).\ filter(Ecosystem.name == ecosystem).\ filter(Package.name == package).\ filter(Version.identifier == version).\ order_by(Analysis.started_at.desc()).\ first() except SQLAlchemyError: rdb.session.rollback() raise
def get(ecosystem, package, version): """Handle the GET REST API call.""" decoded = decode_token() package = urllib.parse.unquote(package) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) package = case_sensitivity_transform(ecosystem, package) result = get_analyses_from_graph(ecosystem, package, version) if result is not None: return result msg = "No data found for {ecosystem} package " \ "{package}/{version}".format(ecosystem=ecosystem, package=package, version=version) raise HTTPError(404, msg)
def ca_validate_input(input_json: Dict, ecosystem: str) -> Tuple[List[Dict], List[Package]]: """Validate CA Input.""" logger.debug('Validating ca input data.') if not input_json: error_msg = "Expected JSON request" raise BadRequest(error_msg) if not isinstance(input_json, dict): error_msg = "Expected list of dependencies in JSON request" raise BadRequest(error_msg) if not check_for_accepted_ecosystem(ecosystem): error_msg: str = f"Ecosystem {ecosystem} is not supported for this request" raise BadRequest(error_msg) if not input_json.get('package_versions'): error_msg: str = "package_versions is missing" raise BadRequest(error_msg) packages_list = [] normalised_input_pkgs = [] for pkg in input_json.get('package_versions'): package = pkg.get("package") version = pkg.get("version") if not all([package, version]): error_msg = "Invalid Input: Package, Version are required." raise BadRequest(error_msg) if (not isinstance(version, str)) or (not isinstance(package, str)): error_msg = "Package version should be string format only." raise BadRequest(error_msg) if not validate_version(version): error_msg = "Package version should not have special characters." raise BadRequest(error_msg) if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) if ecosystem == 'pypi': package = package.lower() packages_list.append({"name": package, "version": version}) normalised_input_pkgs.append(normlize_packages(package, version)) return packages_list, normalised_input_pkgs
def get_analysis_count(self, ecosystem, package): """Get count of previously scheduled analyses for given ecosystem-package. :param ecosystem: str, Ecosystem name :param package: str, Package name :return: analysis count """ if ecosystem == 'maven': package = MavenCoordinates.normalize_str(package) try: count = PostgresBase.session.query(PackageAnalysis).\ join(Package).join(Ecosystem).\ filter(Ecosystem.name == ecosystem).\ filter(Package.name == package).\ count() except SQLAlchemyError: PostgresBase.session.rollback() raise return count
def get_analysis_count(ecosystem, package, version): """Get count of previously scheduled analysis for given EPV triplet. :param ecosystem: str, Ecosystem name :param package: str, Package name :param version: str, Package version :return: analysis count """ if Ecosystem.by_name(PostgresBase.session, ecosystem).is_backed_by(EcosystemBackend.maven): package = MavenCoordinates.normalize_str(package) try: count = PostgresBase.session.query(Analysis).\ join(Version).join(Package).join(Ecosystem).\ filter(Ecosystem.name == ecosystem).\ filter(Package.name == package).\ filter(Version.identifier == version).\ count() except SQLAlchemyError: PostgresBase.session.rollback() raise return count
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result # No data has been found unknown_pkgs = set() unknown_pkgs.add( ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: - DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202 - DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition. - INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch Package details - INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch Package details :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: # Known component for Fabric8 Analytics server_create_component_bookkeeping(ecosystem, package, version, g.decoded_token) metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1": msg = f"No data found for {ecosystem} package {package}/{version} " \ "ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled" return response_template({'error': msg}, 202) if os.environ.get("INVOKE_API_WORKERS", "") == "1": # Trigger the unknown component ingestion. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=True, force=False, force_graph_sync=True) msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \ "The package will be available shortly," \ " please retry after some time." metrics_payload.update({ "status_code": 202, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return response_template({'error': msg}, 202) # No data has been found and INVOKE_API_WORKERS flag is down. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=False, force=False, force_graph_sync=True) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)