def stack_analyses_with_request_id(external_request_id): """Handle stack analyses report fetch api.""" start = time.time() logger.debug("[GET] /stack-analyses/%s", external_request_id) # 1. Build response builder with id and RDB object. sa_response_builder = StackAnalysesResponseBuilder( external_request_id, RdbAnalyses(external_request_id)) # 2. If there was no exception raise, means request is ready to be served. try: data = sa_response_builder.get_response() logger.info('%s took %f seconds for [GET] stack-analyses', external_request_id, time.time() - start) return jsonify(data) except SARBRequestInvalidException as e: raise HTTPError(400, e.args[0]) from e except RDBInvalidRequestException as e: raise HTTPError(404, e.args[0]) from e except RDBServerException as e: raise HTTPError(500, e.args[0]) from e except SARBRequestInprogressException as e: # Avoid HTTPError to ignore sentry reporting for Inprogress request. return jsonify({'error': e.args[0]}), 202 except SARBRequestTimeoutException as e: raise HTTPError(408, e.args[0]) from e
def vulnerability_analysis_post(): """Handle the POST REST API call. Component Analyses Batch is 3 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') try: # Step1: Gather and clean Request packages_list = validate_input(input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_vulnerability_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation = get_known_pkgs(graph_response, packages_list) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(500, msg) from e return jsonify(stack_recommendation), 200
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') if request.user_agent.string == "claircore/crda/RemoteMatcher": try: md5_hash = hashlib.md5( json.dumps(input_json, sort_keys=True).encode('utf-8')).hexdigest() logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem, md5_hash) except Exception as e: logger.error("Exception %s", e) return jsonify({"message": "disabled"}), 404 try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs, input_json.get("ignore", {})) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"]) input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') user_agent = request.headers.get('User-Agent', None) manifest_hash = str(request.headers.get('manifest_hash', None)) request_id = request.headers.get('request_id', None) headers = {"uuid": request.headers.get('uuid', None)} try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, headers.get("uuid"), user_agent, manifest_hash, request_id) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info( stack_recommendation, unknown_pkgs) pkgs_to_ingest = set( map( lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return response_template(stack_recommendation, 202, headers) return response_template(stack_recommendation, 200, headers)
def get_token(): """Return 3Scale tokens with higher rate limit.""" try: # return default key THREESCALE_PREMIUM_USER_KEY = os.getenv('THREESCALE_PREMIUM_USER_KEY') return jsonify({"key": THREESCALE_PREMIUM_USER_KEY}), 200 except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(500, msg) from e
def component_analyses_get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ input_json = { "package_versions": [{ "package": package, "version": version, }] } try: ca_validate_input(input_json, ecosystem) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br if analyses_result is not None: return jsonify(analyses_result) # No data has been found unknown_pkgs = set() unknown_pkgs.add(ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = {"message": f"No data found for {ecosystem} package {package}/{version}"} return jsonify(msg), 404
def wrapper(*args, **kwargs): """Read uuid and decides user type based on its validity.""" # Rule of UUID validation and setting user status :: # ============================================================== # UUID in request | UUID in RDS | RDS User State | User Status # ============================================================== # MISSING | -- NA -- | -- NA -- | FREE # PRESENT | MISSING | -- NA -- | FREE # PRESENT | PRESENT | REGISTERED | REGISTERED # PRESENT | PRESENT | !REGISTERED | FREE # ============================================================== # By default set this to 'freetier' and uuid to None g.user_status = UserStatus.FREETIER g.uuid = None try: header_data = HeaderData(uuid=request.headers.get('uuid', None)) if header_data.uuid: g.uuid = str(header_data.uuid) user = get_user(g.uuid) g.user_status = UserStatus[user.status] except ValidationError as e: raise HTTPError(400, "Not a valid uuid") from e except UserNotFoundException: logger.warning("No User Found corresponding to UUID {}".format(header_data.uuid)) except UserException: logger.warning("Unable to get user status for uuid '{}'".format(header_data.uuid)) logger.debug('For UUID: %s, got user type: %s final uuid: %s', header_data.uuid, g.user_status, g.uuid) return view(*args, **kwargs)
def get_user(user_id): """Endpoint for getting user details.""" if not user_id: raise HTTPError(400, "user id should be present") user = user_utils.get_user(user_id) user_status = user.status if user.status else UserStatus.FREETIER.name return jsonify(user_id=user.user_id, status=user_status)
def create_or_update_user(): """Endpoint for creating or updating user details.""" content = request.json user_id = content.get('user_id') if not user_id: raise HTTPError(400, "user id should be present") snyk_api_token = content.get('snyk_api_token') if not snyk_api_token: raise HTTPError(400, 'snyk api token should be present') if not is_snyk_token_valid(snyk_api_token): raise HTTPError(400, "Invalid API Token") encrypted_api_token = encrypt_api_token(snyk_api_token) user_utils.create_or_update_user(user_id, encrypted_api_token.decode(), "SNYK") return jsonify(user_id=user_id)
def post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"]) input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') headers = {"uuid": request.headers.get('uuid', None)} try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input( input_json, ecosystem) # Step2: Query GraphDB, graph_response = GraphAnalyses.get_batch_ca_data( ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info( stack_recommendation, unknown_pkgs) if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW" ) != "1" and ecosystem != "golang": # Unknown Packages is Present and INGESTION is Enabled logger.debug(unknown_pkgs) unknown_package_flow(ecosystem, unknown_pkgs) return response_template(stack_recommendation, 202, headers) return response_template(stack_recommendation, 200, headers)
def get_user(user_id): """Endpoint for getting user details.""" if not user_id: raise HTTPError(400, "user id should be present") user_status = UserStatus.FREETIER.name user = user_utils.get_user(user_id) if user: user_status = user.status if user.status else UserStatus.FREETIER.name return jsonify(user_id=user_id, status=user_status) else: return jsonify(message='User not found', status=404), 404
def component_analyses_post(): """Handle the POST REST API call. Component Analyses Batch is 4 Step Process: 1. Gather and clean Request. 2. Query GraphDB. 3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow. 4. Handle Unknown Packages and Trigger bayesianApiFlow. """ input_json: Dict = request.get_json() ecosystem: str = input_json.get('ecosystem') try: # Step1: Gather and clean Request packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem) # Step2: Get aggregated CA data from Query GraphDB, graph_response = get_batch_ca_data(ecosystem, packages_list) # Step3: Build Unknown packages and Generates Stack Recommendation. stack_recommendation, unknown_pkgs = get_known_unknown_pkgs( ecosystem, graph_response, normalised_input_pkgs) except BadRequest as br: logger.error(br) raise HTTPError(400, str(br)) from br except Exception as e: msg = "Internal Server Exception. Please contact us if problem persists." logger.error(e) raise HTTPError(400, msg) from e create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers) # Step4: Handle Unknown Packages if unknown_pkgs: stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs) pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package, version=pkg.version), unknown_pkgs)) logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest) unknown_package_flow(ecosystem, pkgs_to_ingest) return jsonify(stack_recommendation), 202 return jsonify(stack_recommendation), 200
def stack_analyses(): """Handle request to trigger a new stack analyses report. GET method would raise error to provide missing request id to the user. """ logger.debug('[%s] /stack-analyses accessed', request.method) start = time.time() if request.method == 'GET': raise HTTPError(400, error="Request id missing") sa_post_request = None try: # 1. Validate and build request object. sa_post_request = StackAnalysesPostRequest(**request.form, **request.files) except ValidationError as e: # 2. Check of invalid params and raise exception. error_message = 'Validation error(s) in the request.' for error in e.errors(): error_message += ' {}.'.format(error['msg']) logger.exception(error_message) raise HTTPError(400, error=error_message) from e # 3. Initiate stack analyses object sa = StackAnalyses(sa_post_request) # 4. Post request try: data = sa.post_request() logger.info('%s took %f seconds for [POST] stack-analyses', data['id'], time.time() - start) return jsonify(data) except SAInvalidInputException as e: raise HTTPError(400, e.args[0]) from e except BackboneServerException as e: raise HTTPError(500, e.args[0]) except RDBSaveException as e: raise HTTPError(500, e.args[0])
def error(): """Implement the endpoint used by httpd, which redirects its errors to it.""" try: status = int(os.getenv("REDIRECT_STATUS")) except Exception: # if there's an exception, it means that a client accessed this directly; # in this case, we want to make it look like the endpoint is not here return api_404_handler("/api/v2/") msg = 'Unknown error' if status == 401: msg = 'Authentication failed' elif status == 405: msg = 'Method not allowed for this endpoint' raise HTTPError(status, msg)
def wrapper(*args, **kwargs): """Read uuid and decides user type based on its validity.""" # Rule of UUID validation and setting user status :: # ============================================================== # UUID in request | UUID in RDS | RDS User State | User Status # ============================================================== # MISSING | -- NA -- | -- NA -- | FREE # PRESENT | MISSING | -- NA -- | FREE # PRESENT | PRESENT | REGISTERED | REGISTERED # PRESENT | PRESENT | !REGISTERED | FREE # ============================================================== # By default set this to 'freetier' and uuid to None g.user_status = UserStatus.FREETIER g.uuid = None try: header_data = HeaderData(uuid=request.headers.get('uuid', None)) if header_data.uuid: g.uuid = str(header_data.uuid) if ENABLE_USER_CACHING: logger.info( "Getting user details from cache for user = %s", g.uuid) user = get_user_from_cache(g.uuid) if user: g.user_status = UserStatus["REGISTERED"] logger.info( 'For UUID: %s, got user type: %s final uuid: %s', header_data.uuid, g.user_status, g.uuid) else: logger.info( 'For UUID: %s, user not found type: %s final uuid: %s', header_data.uuid, g.user_status, g.uuid) else: logger.info("Getting user details from RDS.") user = get_user(g.uuid) if user: g.user_status = UserStatus[user.status] logger.info( 'For UUID: %s, got user type: %s final uuid: %s', header_data.uuid, g.user_status, g.uuid) except ValidationError as e: raise HTTPError(400, "Not a valid uuid") from e except UserException: logger.warning("Unable to get user status for uuid '{}'".format( header_data.uuid)) return view(*args, **kwargs)
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: - DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202 - DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition. - INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch Package details - INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch Package details :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: # Known component for Fabric8 Analytics server_create_component_bookkeeping(ecosystem, package, version, g.decoded_token) metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1": msg = f"No data found for {ecosystem} package {package}/{version} " \ "ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled" return response_template({'error': msg}, 202) if os.environ.get("INVOKE_API_WORKERS", "") == "1": # Trigger the unknown component ingestion. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=True, force=False, force_graph_sync=True) msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \ "The package will be available shortly," \ " please retry after some time." metrics_payload.update({ "status_code": 202, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return response_template({'error': msg}, 202) # No data has been found and INVOKE_API_WORKERS flag is down. server_create_analysis(ecosystem, package, version, user_profile=g.decoded_token, api_flow=False, force=False, force_graph_sync=True) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)
def get(ecosystem, package, version): """Handle the GET REST API call. Component Analyses: - If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response. - If package is not Known: Call Util's function to trigger ingestion flow. :return: JSON Response """ st = time.time() # Analytics Data metrics_payload = { "pid": os.getpid(), "hostname": HOSTNAME, "endpoint": request.endpoint, "request_method": "GET", "ecosystem": ecosystem, "package": package, "version": version } response_template = namedtuple("response_template", ["message", "status"]) logger.info("Executed v2 API") package = urllib.parse.unquote(package) if re.findall('[!@#$%^&*()]', version): # Version should not contain special Characters. return response_template( { 'error': "Package version should not have special characters." }, 400) if not check_for_accepted_ecosystem(ecosystem): msg = f"Ecosystem {ecosystem} is not supported for this request" raise HTTPError(400, msg) if ecosystem == 'maven': try: package = MavenCoordinates.normalize_str(package) except ValueError: msg = f"Invalid maven format - {package}" metrics_payload.update({ "status_code": 400, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(400, msg) package = case_sensitivity_transform(ecosystem, package) # Perform Component Analyses on Vendor specific Graph Edge. analyses_result = ComponentAnalyses( ecosystem, package, version).get_component_analyses_response() if analyses_result is not None: metrics_payload.update({ "status_code": 200, "value": time.time() - st }) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) return analyses_result # No data has been found unknown_pkgs = set() unknown_pkgs.add( ingestion_utils.Package(package=package, version=version)) unknown_package_flow(ecosystem, unknown_pkgs) msg = f"No data found for {ecosystem} package {package}/{version}" metrics_payload.update({"status_code": 404, "value": time.time() - st}) _session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus", json=metrics_payload) raise HTTPError(404, msg)