def stack_analyses_with_request_id(external_request_id):
"""Handle stack analyses report fetch api."""
start = time.time()
logger.debug("[GET] /stack-analyses/%s", external_request_id)
# 1. Build response builder with id and RDB object.
sa_response_builder = StackAnalysesResponseBuilder(
external_request_id, RdbAnalyses(external_request_id))
# 2. If there was no exception raise, means request is ready to be served.
try:
data = sa_response_builder.get_response()
logger.info('%s took %f seconds for [GET] stack-analyses',
external_request_id,
time.time() - start)
return jsonify(data)
except SARBRequestInvalidException as e:
raise HTTPError(400, e.args[0]) from e
except RDBInvalidRequestException as e:
raise HTTPError(404, e.args[0]) from e
except RDBServerException as e:
raise HTTPError(500, e.args[0]) from e
except SARBRequestInprogressException as e:
# Avoid HTTPError to ignore sentry reporting for Inprogress request.
return jsonify({'error': e.args[0]}), 202
except SARBRequestTimeoutException as e:
raise HTTPError(408, e.args[0]) from e
def vulnerability_analysis_post():
"""Handle the POST REST API call.
Component Analyses Batch is 3 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list = validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_vulnerability_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation = get_known_pkgs(graph_response, packages_list)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(500, msg) from e
return jsonify(stack_recommendation), 200
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def get_token():
"""Return 3Scale tokens with higher rate limit."""
try:
# return default key
THREESCALE_PREMIUM_USER_KEY = os.getenv('THREESCALE_PREMIUM_USER_KEY')
return jsonify({"key": THREESCALE_PREMIUM_USER_KEY}), 200
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(500, msg) from e
def component_analyses_get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
input_json = {
"package_versions": [{
"package": package,
"version": version,
}]
}
try:
ca_validate_input(input_json, ecosystem)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
if analyses_result is not None:
return jsonify(analyses_result)
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = {"message": f"No data found for {ecosystem} package {package}/{version}"}
return jsonify(msg), 404
def wrapper(*args, **kwargs):
"""Read uuid and decides user type based on its validity."""
# Rule of UUID validation and setting user status ::
# ==============================================================
# UUID in request | UUID in RDS | RDS User State | User Status
# ==============================================================
# MISSING | -- NA -- | -- NA -- | FREE
# PRESENT | MISSING | -- NA -- | FREE
# PRESENT | PRESENT | REGISTERED | REGISTERED
# PRESENT | PRESENT | !REGISTERED | FREE
# ==============================================================
# By default set this to 'freetier' and uuid to None
g.user_status = UserStatus.FREETIER
g.uuid = None
try:
header_data = HeaderData(uuid=request.headers.get('uuid', None))
if header_data.uuid:
g.uuid = str(header_data.uuid)
user = get_user(g.uuid)
g.user_status = UserStatus[user.status]
except ValidationError as e:
raise HTTPError(400, "Not a valid uuid") from e
except UserNotFoundException:
logger.warning("No User Found corresponding to UUID {}".format(header_data.uuid))
except UserException:
logger.warning("Unable to get user status for uuid '{}'".format(header_data.uuid))
logger.debug('For UUID: %s, got user type: %s final uuid: %s',
header_data.uuid, g.user_status, g.uuid)
return view(*args, **kwargs)
def get_user(user_id):
"""Endpoint for getting user details."""
if not user_id:
raise HTTPError(400, "user id should be present")
user = user_utils.get_user(user_id)
user_status = user.status if user.status else UserStatus.FREETIER.name
return jsonify(user_id=user.user_id, status=user_status)
def create_or_update_user():
"""Endpoint for creating or updating user details."""
content = request.json
user_id = content.get('user_id')
if not user_id:
raise HTTPError(400, "user id should be present")
snyk_api_token = content.get('snyk_api_token')
if not snyk_api_token:
raise HTTPError(400, 'snyk api token should be present')
if not is_snyk_token_valid(snyk_api_token):
raise HTTPError(400, "Invalid API Token")
encrypted_api_token = encrypt_api_token(snyk_api_token)
user_utils.create_or_update_user(user_id, encrypted_api_token.decode(), "SNYK")
return jsonify(user_id=user_id)
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Query GraphDB,
graph_response = GraphAnalyses.get_batch_ca_data(
ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW"
) != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def get_user(user_id):
"""Endpoint for getting user details."""
if not user_id:
raise HTTPError(400, "user id should be present")
user_status = UserStatus.FREETIER.name
user = user_utils.get_user(user_id)
if user:
user_status = user.status if user.status else UserStatus.FREETIER.name
return jsonify(user_id=user_id, status=user_status)
else:
return jsonify(message='User not found', status=404), 404
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def stack_analyses():
"""Handle request to trigger a new stack analyses report.
GET method would raise error to provide missing request id to the user.
"""
logger.debug('[%s] /stack-analyses accessed', request.method)
start = time.time()
if request.method == 'GET':
raise HTTPError(400, error="Request id missing")
sa_post_request = None
try:
# 1. Validate and build request object.
sa_post_request = StackAnalysesPostRequest(**request.form,
**request.files)
except ValidationError as e:
# 2. Check of invalid params and raise exception.
error_message = 'Validation error(s) in the request.'
for error in e.errors():
error_message += ' {}.'.format(error['msg'])
logger.exception(error_message)
raise HTTPError(400, error=error_message) from e
# 3. Initiate stack analyses object
sa = StackAnalyses(sa_post_request)
# 4. Post request
try:
data = sa.post_request()
logger.info('%s took %f seconds for [POST] stack-analyses', data['id'],
time.time() - start)
return jsonify(data)
except SAInvalidInputException as e:
raise HTTPError(400, e.args[0]) from e
except BackboneServerException as e:
raise HTTPError(500, e.args[0])
except RDBSaveException as e:
raise HTTPError(500, e.args[0])
def error():
"""Implement the endpoint used by httpd, which redirects its errors to it."""
try:
status = int(os.getenv("REDIRECT_STATUS"))
except Exception:
# if there's an exception, it means that a client accessed this directly;
# in this case, we want to make it look like the endpoint is not here
return api_404_handler("/api/v2/")
msg = 'Unknown error'
if status == 401:
msg = 'Authentication failed'
elif status == 405:
msg = 'Method not allowed for this endpoint'
raise HTTPError(status, msg)
def wrapper(*args, **kwargs):
"""Read uuid and decides user type based on its validity."""
# Rule of UUID validation and setting user status ::
# ==============================================================
# UUID in request | UUID in RDS | RDS User State | User Status
# ==============================================================
# MISSING | -- NA -- | -- NA -- | FREE
# PRESENT | MISSING | -- NA -- | FREE
# PRESENT | PRESENT | REGISTERED | REGISTERED
# PRESENT | PRESENT | !REGISTERED | FREE
# ==============================================================
# By default set this to 'freetier' and uuid to None
g.user_status = UserStatus.FREETIER
g.uuid = None
try:
header_data = HeaderData(uuid=request.headers.get('uuid', None))
if header_data.uuid:
g.uuid = str(header_data.uuid)
if ENABLE_USER_CACHING:
logger.info(
"Getting user details from cache for user = %s",
g.uuid)
user = get_user_from_cache(g.uuid)
if user:
g.user_status = UserStatus["REGISTERED"]
logger.info(
'For UUID: %s, got user type: %s final uuid: %s',
header_data.uuid, g.user_status, g.uuid)
else:
logger.info(
'For UUID: %s, user not found type: %s final uuid: %s',
header_data.uuid, g.user_status, g.uuid)
else:
logger.info("Getting user details from RDS.")
user = get_user(g.uuid)
if user:
g.user_status = UserStatus[user.status]
logger.info(
'For UUID: %s, got user type: %s final uuid: %s',
header_data.uuid, g.user_status, g.uuid)
except ValidationError as e:
raise HTTPError(400, "Not a valid uuid") from e
except UserException:
logger.warning("Unable to get user status for uuid '{}'".format(
header_data.uuid))
return view(*args, **kwargs)
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known:
- DISABLE_UNKNOWN_PACKAGE_FLOW flag is 1: Skips the unknown package and returns 202
- DISABLE_UNKONWN_PACKAGE_FLOW flag is 0: Than checks below condition.
- INVOKE_API_WORKERS flag is 1: Trigger bayesianApiFlow to fetch
Package details
- INVOKE_API_WORKERS flag is 0: Trigger bayesianFlow to fetch
Package details
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
# Known component for Fabric8 Analytics
server_create_component_bookkeeping(ecosystem, package, version,
g.decoded_token)
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
elif os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "1":
msg = f"No data found for {ecosystem} package {package}/{version} " \
"ingetion flow skipped as DISABLE_UNKNOWN_PACKAGE_FLOW is enabled"
return response_template({'error': msg}, 202)
if os.environ.get("INVOKE_API_WORKERS", "") == "1":
# Trigger the unknown component ingestion.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=True,
force=False,
force_graph_sync=True)
msg = f"Package {ecosystem}/{package}/{version} is unavailable. " \
"The package will be available shortly," \
" please retry after some time."
metrics_payload.update({
"status_code": 202,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return response_template({'error': msg}, 202)
# No data has been found and INVOKE_API_WORKERS flag is down.
server_create_analysis(ecosystem,
package,
version,
user_profile=g.decoded_token,
api_flow=False,
force=False,
force_graph_sync=True)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
def get(ecosystem, package, version):
"""Handle the GET REST API call.
Component Analyses:
- If package is Known (exists in GraphDB (Snyk Edge) returns Json formatted response.
- If package is not Known: Call Util's function to trigger ingestion flow.
:return:
JSON Response
"""
st = time.time()
# Analytics Data
metrics_payload = {
"pid": os.getpid(),
"hostname": HOSTNAME,
"endpoint": request.endpoint,
"request_method": "GET",
"ecosystem": ecosystem,
"package": package,
"version": version
}
response_template = namedtuple("response_template",
["message", "status"])
logger.info("Executed v2 API")
package = urllib.parse.unquote(package)
if re.findall('[!@#$%^&*()]', version):
# Version should not contain special Characters.
return response_template(
{
'error':
"Package version should not have special characters."
}, 400)
if not check_for_accepted_ecosystem(ecosystem):
msg = f"Ecosystem {ecosystem} is not supported for this request"
raise HTTPError(400, msg)
if ecosystem == 'maven':
try:
package = MavenCoordinates.normalize_str(package)
except ValueError:
msg = f"Invalid maven format - {package}"
metrics_payload.update({
"status_code": 400,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(400, msg)
package = case_sensitivity_transform(ecosystem, package)
# Perform Component Analyses on Vendor specific Graph Edge.
analyses_result = ComponentAnalyses(
ecosystem, package, version).get_component_analyses_response()
if analyses_result is not None:
metrics_payload.update({
"status_code": 200,
"value": time.time() - st
})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
return analyses_result
# No data has been found
unknown_pkgs = set()
unknown_pkgs.add(
ingestion_utils.Package(package=package, version=version))
unknown_package_flow(ecosystem, unknown_pkgs)
msg = f"No data found for {ecosystem} package {package}/{version}"
metrics_payload.update({"status_code": 404, "value": time.time() - st})
_session.post(url=METRICS_SERVICE_URL + "/api/v1/prometheus",
json=metrics_payload)
raise HTTPError(404, msg)
