def send_request(context,
ecosystem=None,
manifest=None,
token='without',
valid='valid',
trans="without",
uuid="without",
invalid="without"):
"""Send the ecosystem package manifest file to the stack analysis v2."""
# Ecosystem is mandatory
assert ecosystem is not None
# Convert token text into a valid bool
with_user_key = parse_token_clause(token)
is_user_registered = parse_token_clause(uuid)
is_invalid = parse_token_clause(invalid)
print(is_invalid)
# Convert valid clause to bool
with_valid_user_key = parse_valid_clause(valid)
with_transitives = parse_token_clause(trans)
post_request(context, ecosystem, manifest, with_user_key,
with_valid_user_key, with_transitives, is_user_registered,
is_invalid)
def finish_analysis_for_component(context,
ecosystem,
component,
version,
token='without'):
"""Try to wait for the component analysis to be finished.
Current API implementation returns just two HTTP codes:
200 OK : analysis is already finished
404 NOT FOUND: analysis is started or is in progress
"""
timeout = context.component_analysis_timeout # in seconds
sleep_amount = 10 # we don't have to overload the API with too many calls
use_token = parse_token_clause(token)
url = component_analysis_url(context, ecosystem, component, version)
for _ in range(timeout // sleep_amount):
if use_token:
status_code = requests.get(
url, headers=authorization(context)).status_code
else:
status_code = requests.get(url).status_code
if status_code == 200:
break
elif status_code != 404:
raise Exception('Bad HTTP status code {c}'.format(c=status_code))
time.sleep(sleep_amount)
else:
raise Exception('Timeout waiting for the component analysis results')
def post_input_to_user_feedback(context, is_valid, endpoint, token):
"""Send feedback to user feedback endpoint."""
use_token = parse_token_clause(token)
api_url = urljoin(context.coreapi_url, endpoint)
if is_valid == "valid":
data = {
"request_id": "test_id",
"feedback": [{
"ques": "what",
"ans": "got it"
}]
}
elif is_valid == "invalid":
data = {"foo": "x", "bar": "y", "baz": []}
elif is_valid == "incomplete":
data = {"request_id": "test_id"}
elif is_valid == "empty":
data = {}
else:
data = None
if use_token:
response = requests.post(api_url,
json=data,
headers=authorization(context))
else:
response = requests.post(api_url, json=data)
context.response = response
def send_the_file_for_license_analysis(context, filename, token="without"):
"""Test step to send the selected file to the license analysis service."""
use_token = parse_token_clause(token)
send_payload_to_license_analysis(context,
LICENSE_ANALYSIS_PAYLOAD_DIRECTORY,
filename, LICENSE_RECOMMENDER_ENDPOINT,
use_token)
def send_the_file_for_stack_license_analysis(context,
filename,
token="without"):
"""Test step to send the payload to the stack analysis endpoint of license analysis service."""
use_token = parse_token_clause(token)
send_payload_to_license_analysis(context, STACK_LICENSE_PAYLOAD_DIRECTORY,
filename, STACK_LICENSE_ENDPOINT,
use_token)
def access_gemini_url(context, endpoint, token="without"):
"""Access the Gemini service API using the HTTP GET method."""
url = urljoin(context.gemini_api_url, endpoint)
use_token = parse_token_clause(token)
headers = {}
if use_token:
headers = authorization(context)
context.response = requests.get(url, headers=headers)
def clean_all_failed_jobs(context, token="without"):
"""Perform API call to clean up all failed jobs."""
url = "{url}api/v1/jobs/clean-failed".format(url=context.jobs_api_url)
use_token = parse_token_clause(token)
if use_token:
context.response = requests.delete(
url, headers=jobs_api_authorization(context))
else:
context.response = requests.delete(url)
def perform_post_job(context, metadata, state, token="without"):
"""Perform API call to create a new job using the provided metadata.
The token parameter can be set to 'with', 'without', or 'using'.
"""
filename = job_metadata_filename(metadata)
endpoint = flow_sheduling_endpoint(context, state)
use_token = parse_token_clause(token)
send_json_file_to_job_api(context, endpoint, filename, use_token)
def maven_manifest_stack_analysis(context,
manifest,
version=3,
token="without"):
"""Send the Maven package manifest file to the stack analysis."""
endpoint = stack_analysis_endpoint(context, version)
use_token = parse_token_clause(token)
send_manifest_to_stack_analysis(context, manifest, 'pom.xml', endpoint,
use_token)
def python_manifest_stack_analysis(context,
manifest,
version=3,
token="without"):
"""Send the Python package manifest file to the stack analysis."""
endpoint = stack_analysis_endpoint(context, version)
use_token = parse_token_clause(token)
send_manifest_to_stack_analysis(context, manifest, 'requirements.txt',
endpoint, use_token)
def delete_job(context, job_id=None, token="without"):
"""Perform API call to delete a job with given ID."""
job_id = get_unique_job_id(context, job_id)
endpoint = job_endpoint(context, job_id)
use_token = parse_token_clause(token)
if use_token:
context.response = requests.delete(
endpoint, headers=jobs_api_authorization(context))
else:
context.response = requests.delete(endpoint)
def logout_from_the_jobs_service(context, token='without'):
"""Call API to logout from the job service."""
url = "{jobs_api_url}api/v1/logout".format(
jobs_api_url=context.jobs_api_url)
use_token = parse_token_clause(token)
if use_token:
headers = jobs_api_authorization(context)
context.response = requests.put(url, headers)
else:
context.response = requests.put(url)
def set_job_status(context, job_id, status, token="without"):
"""Perform API call to set job status."""
endpoint = job_endpoint(context, job_id)
url = "{endpoint}?state={status}".format(endpoint=endpoint, status=status)
use_token = parse_token_clause(token)
if use_token:
context.response = requests.put(
url, headers=jobs_api_authorization(context))
else:
context.response = requests.put(url)
def npm_manifest_stack_analysis(context,
manifest,
version=3,
user_key="without",
rate=1):
"""Send the NPM package manifest file to the stack analyses."""
endpoint = threescale_preview_endpoint_url(context, 'stack-analyses')
use_user_key = parse_token_clause(user_key)
send_manifest_to_stack_analyses(context, manifest, 'package.json',
endpoint, use_user_key, rate)
def start_component_analysis(context,
ecosystem,
package,
version,
user_key,
rate=1):
"""Analyse the given component."""
use_user_key = parse_token_clause(user_key)
perform_component_analysis(context, ecosystem, package, version,
use_user_key, rate)
def wait_for_completion(context, token='without'):
"""Try to wait for the stack analysis to be finished.
This step assumes that stack analysis has been started previously and
thus that the request id is known
Current API implementation returns just three HTTP codes:
200 OK : analysis is already finished
202 Accepted: analysis is started or is in progress (or other state!)
401 UNAUTHORIZED : missing or improper user key
408 Timeout: Request timeout.
"""
context.duration = None
start_time = time.time()
timeout = context.stack_analysis_timeout # in seconds
sleep_amount = 10 # we don't have to overload the API with too many calls
with_user_key = parse_token_clause(token)
id = context.response.json().get('id')
context.stack_analysis_id = id
logger.debug('SA V2 Request id: {}'.format(id))
url = urljoin(get_endpoint(context), id)
logger.debug('Get API url: {}'.format(url))
for _ in range(timeout // sleep_amount):
if with_user_key:
params = {'user_key': context.three_scale_preview_user_key}
context.response = requests.get(url, params=params)
else:
context.response = requests.get(url)
status_code = context.response.status_code
logger.debug('status_code: {}'.format(status_code))
if status_code == 200:
break
# 429 (Rate Limit) code should be checked later
elif status_code == 429:
break
# 403 code should be checked later
elif status_code == 403:
break
# 401 code should be checked later
elif status_code == 401:
break
elif status_code != 202:
raise Exception('Bad HTTP status code {c}'.format(c=status_code))
time.sleep(sleep_amount)
else:
raise Exception('Timeout waiting for the stack analysis results')
end_time = time.time()
# compute the duration
# plase note that duration==None in case of any errors (which is to be expected)
context.duration = end_time - start_time
def npm_manifest_stack_analysis(context, manifest, version=3, token="without"):
"""Send the NPM package manifest file to the stack analysis."""
endpoint = stack_analysis_endpoint(context, version)
use_token = parse_token_clause(token)
send_manifest_to_stack_analysis(context,
manifest,
'package.json',
endpoint,
use_token,
ecosystem='npm',
origin='vscode')
def post_input_to_user_feedback(context, is_valid, endpoint, token):
"""Send feedback to user feedback endpoint."""
use_token = parse_token_clause(token)
api_url = urljoin(context.coreapi_url, endpoint)
data = generate_data_for_user_feedback(is_valid)
if use_token:
response = requests.post(api_url, json=data,
headers=authorization(context))
else:
response = requests.post(api_url, json=data)
context.response = response
def list_of_jobs(context, type=None, token=None):
"""Read list of jobs via job API."""
endpoint = job_endpoint(context)
if type is not None:
endpoint += "?job_type=" + type
use_token = parse_token_clause(token)
if use_token:
context.response = requests.get(
endpoint, headers=jobs_api_authorization(context))
else:
context.response = requests.get(endpoint)
def scale_process_deps_file(context,
ecosystem,
manifest,
origin,
rate=1,
user_key="without"):
"""Test stack analyses of an ecosystem specific dependencies file from an integration point."""
endpoint = threescale_preview_endpoint_url(context, 'stack-analyses')
use_user_key = parse_token_clause(user_key)
test_stack_analyses_with_deps_file(context, manifest,
endpoint, use_user_key, rate,
ecosystem.lower(), origin)
def set_job_service_status(context, status=None, token="without"):
"""Perform API call to set or reset job service status."""
url = "{jobs_api_url}api/v1/service/state".format(
jobs_api_url=context.jobs_api_url)
use_token = parse_token_clause(token)
if status is not None:
url = "{url}?state={status}".format(url=url, status=status)
if use_token:
context.response = requests.put(
url, headers=jobs_api_authorization(context))
else:
context.response = requests.put(url)
def read_analysis_for_component(context,
ecosystem,
component,
version,
token='without'):
"""Read component analysis (or an error message) for the selected ecosystem."""
url = component_analysis_url(context, ecosystem, component, version)
use_token = parse_token_clause(token)
if use_token:
context.response = requests.get(url, headers=authorization(context))
else:
context.response = requests.get(url)
def maven_new_manifest_stack_analysis(context,
manifest,
version=3,
token="without"):
"""Send the Maven package manifest file to the stack analysis."""
endpoint = stack_analysis_endpoint(context, version)
use_token = parse_token_clause(token)
send_manifest_to_stack_analysis(context,
manifest,
'dependencies.txt',
endpoint,
use_token,
ecosystem='maven',
origin='vscode')
def wait_for_stack_analysis_completion(context, version=3, token="without"):
"""Try to wait for the stack analysis to be finished.
This step assumes that stack analysis has been started previously and
thus that the job ID is known
Current API implementation returns just three HTTP codes:
200 OK : analysis is already finished
202 Accepted: analysis is started or is in progress (or other state!)
401 UNAUTHORIZED : missing or improper authorization token
"""
context.duration = None
start_time = time.time()
timeout = context.stack_analysis_timeout # in seconds
sleep_amount = 10 # we don't have to overload the API with too many calls
use_token = parse_token_clause(token)
id = context.response.json().get("id")
context.stack_analysis_id = id
# log.info("REQUEST ID: {}\n\n".format(context.stack_analysis_id))
url = urljoin(stack_analysis_endpoint(context, version), id)
# log.info("RECOMMENDER API URL: {}\n\n".format(url))
for _ in range(timeout // sleep_amount):
if use_token:
context.response = requests.get(url,
headers=authorization(context))
else:
context.response = requests.get(url)
status_code = context.response.status_code
# log.info("%r" % context.response.json())
if status_code == 200:
json_resp = context.response.json()
if contains_alternate_node(json_resp):
# log.info('Recommendation found')
break
# 401 code should be checked later
elif status_code == 401:
break
elif status_code != 202:
raise Exception('Bad HTTP status code {c}'.format(c=status_code))
time.sleep(sleep_amount)
else:
raise Exception('Timeout waiting for the stack analysis results')
end_time = time.time()
# compute the duration
# plase note that duration==None in case of any errors (which is to be expected)
context.duration = end_time - start_time
def perform_post_job_with_state(context,
metadata,
job_id,
state,
token="without"):
"""Perform API call to create a new job.
The new job is created using the provided metadata and set a job
to given state. The token parameter can be set to 'with', 'without', or
'using'.
"""
filename = job_metadata_filename(metadata)
job_id = get_unique_job_id(context, job_id)
endpoint = flow_sheduling_endpoint(context, state, job_id)
use_token = parse_token_clause(token)
send_json_file_to_job_api(context, endpoint, filename, use_token)
def maven_manifest_stack_analysis(context,
manifest,
version=3,
user_key="without",
rate=1):
"""Send the NPM package manifest file to the stack analyses."""
endpoint = threescale_preview_endpoint_url(context, 'stack-analyses')
use_user_key = parse_token_clause(user_key)
send_manifest_to_stack_analyses(context,
manifest,
'dependencies.txt',
endpoint,
use_user_key,
rate,
ecosystem='maven',
origin='vscode')
def start_rate_limit_requests(context,
ecosystem,
manifest,
rate,
user_key='with',
valid='valid'):
"""Multiple SA request within given time window to generate 429 (rate limit)."""
# Convert token text into a valid bool
with_user_key = parse_token_clause(user_key)
# Convert valid clause to bool
with_valid_user_key = parse_valid_clause(valid)
# Perform loop to send requested number of request and compute duration.
check_rate_limit(context, ecosystem, manifest, rate, with_user_key,
with_valid_user_key)
def wait_for_stack_analysis_completion(context, user_key="without", rate=1):
"""Try to wait for the stack analysis to be finished.
This step assumes that stack analysis has been started previously and
thus that the job ID is known
Current API implementation returns just three HTTP codes:
200 OK : analysis is already finished
202 Accepted: analysis is started or is in progress (or other state!)
403 UNAUTHORIZED : missing or improper user_key
"""
context.duration = None
start_time = time.time()
timeout = context.stack_analysis_timeout # in seconds
sleep_amount = 15 # we don't have to overload the API with too many calls
use_user_key = parse_token_clause(user_key)
id = context.response.json().get("id")
context.stack_analysis_id = id
url = urljoin(threescale_preview_endpoint_url(context, 'stack-analyses'),
id)
for _ in range((timeout // sleep_amount) + rate):
if use_user_key:
context.response = requests.get(
url, params={'user_key': context.three_scale_preview_user_key})
else:
context.response = requests.get(url)
status_code = context.response.status_code
if status_code == 200:
json_resp = context.response.json()
if contains_alternate_node(json_resp) and rate <= 1:
break
# 429 (Rate Limit) code should be checked later
elif status_code == 429:
break
# 403 code should be checked later
elif status_code == 403:
break
elif status_code != 202:
raise Exception('Bad HTTP status code {c}'.format(c=status_code))
pause_between_stack_analysis_requests(sleep_amount, rate)
else:
raise Exception('Timeout waiting for the stack analysis results')
end_time = time.time()
context.duration = end_time - start_time
def access_analyses_report(context,
ecosystem,
from_date=None,
to_date=None,
token="without"):
"""Perform API call to get analyses report for selected ecosystem."""
use_token = parse_token_clause(token)
url = "{url}api/v1/debug/analyses-report?ecosystem={ecosystem}".format(
url=context.jobs_api_url, ecosystem=ecosystem)
if from_date is not None:
url += "&from_date=" + from_date
if to_date is not None:
url += "&to_date=" + to_date
if use_token:
headers = jobs_api_authorization(context)
context.response = requests.get(url, headers=headers)
else:
context.response = requests.get(url)
def send_request(context,
ecosystem=None,
manifest=None,
token='without',
valid='valid'):
"""Send the ecosystem package manifest file to the stack analysis v2."""
# Ecosystem is mandatory
assert ecosystem is not None
# Convert token text into a valid bool
with_user_key = parse_token_clause(token)
# Convert valid clause to bool
with_valid_user_key = parse_valid_clause(valid)
# Send SA request
post_request(context, ecosystem, manifest, with_user_key,
with_valid_user_key)
