def test_get_batch_ca_data_both(self, _mocksemver, _mockpseudo):
"""Test Ca batch data."""
_mocksemver.return_value = self.gremlin_batch
_mockpseudo.return_value = self.gremlin_batch
packages = [
{
'name': 'django',
'version': '1.1',
'given_version': '1.1',
'is_pseudo_version': False
},
{
'name': 'github.com/crda/test/package1',
'version': '0.0.0-20180902000632-abcd4321dcba',
'given_version': 'v0.0.0-20180902000632-abcd4321dcba',
'is_pseudo_version': True
}
]
result = get_batch_ca_data('golang', packages)
self.assertIsInstance(result, dict)
self.assertIn('result', result)
self.assertIsInstance(result.get('result'), dict)
self.assertIn('requestId', result)
self.assertIsInstance(result.get('requestId'), str)
self.assertIn('status', result)
self.assertIsInstance(result.get('status'), dict)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
if request.user_agent.string == "claircore/crda/RemoteMatcher":
try:
md5_hash = hashlib.md5(
json.dumps(input_json,
sort_keys=True).encode('utf-8')).hexdigest()
logger.info("Ecosystem: %s => body md5 hash: %s", ecosystem,
md5_hash)
except Exception as e:
logger.error("Exception %s", e)
return jsonify({"message": "disabled"}), 404
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs,
input_json.get("ignore", {}))
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args,
request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation,
unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template",
["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
user_agent = request.headers.get('User-Agent', None)
manifest_hash = str(request.headers.get('manifest_hash', None))
request_id = request.headers.get('request_id', None)
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(
input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list,
headers.get("uuid"), user_agent,
manifest_hash, request_id)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(
stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(
map(
lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def test_get_batch_ca_data_pseudo_version(self, _mockca):
"""Test Ca batch data."""
_mockca.return_value = self.gremlin_batch
packages = [{
'name': 'github.com/crda/test/package1',
'version': '0.0.0-20180902000632-abcd4321dcba',
'given_version': 'v0.0.0-20180902000632-abcd4321dcba',
'is_pseudo_version': True
}]
result = get_batch_ca_data('golang', packages)
self.assertIsInstance(result, dict)
self.assertIn('result', result)
self.assertIsInstance(result.get('result'), dict)
def test_get_batch_ca_data_semver(self, _mockca):
"""Test Ca batch data."""
_mockca.return_value = self.gremlin_batch
packages = [{
'name': 'django',
'version': '1.1',
'given_version': '1.1',
'is_pseudo_version': False
}]
result = get_batch_ca_data('pypi', packages)
self.assertIsInstance(result, dict)
self.assertIn('result', result)
self.assertIsInstance(result.get('result'), dict)
def component_analyses_post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
create_component_bookkeeping(ecosystem, packages_list, request.args, request.headers)
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
pkgs_to_ingest = set(map(lambda pkg: ingestion_utils.Package(package=pkg.package,
version=pkg.version),
unknown_pkgs))
logger.debug("Unknown ingestion triggered for %s", pkgs_to_ingest)
unknown_package_flow(ecosystem, pkgs_to_ingest)
return jsonify(stack_recommendation), 202
return jsonify(stack_recommendation), 200
def post():
"""Handle the POST REST API call.
Component Analyses Batch is 4 Step Process:
1. Gather and clean Request.
2. Query GraphDB.
3. Build Stack Recommendation and Build Unknown Packages and Trigger componentApiFlow.
4. Handle Unknown Packages and Trigger bayesianApiFlow.
"""
response_template: Tuple = namedtuple("response_template", ["message", "status", "headers"])
input_json: Dict = request.get_json()
ecosystem: str = input_json.get('ecosystem')
headers = {"uuid": request.headers.get('uuid', None)}
try:
# Step1: Gather and clean Request
packages_list, normalised_input_pkgs = ca_validate_input(input_json, ecosystem)
# Step2: Get aggregated CA data from Query GraphDB,
graph_response = get_batch_ca_data(ecosystem, packages_list)
# Step3: Build Unknown packages and Generates Stack Recommendation.
stack_recommendation, unknown_pkgs = get_known_unknown_pkgs(
ecosystem, graph_response, normalised_input_pkgs)
except BadRequest as br:
logger.error(br)
raise HTTPError(400, str(br)) from br
except Exception as e:
msg = "Internal Server Exception. Please contact us if problem persists."
logger.error(e)
raise HTTPError(400, msg) from e
# Step4: Handle Unknown Packages
if unknown_pkgs:
stack_recommendation = add_unknown_pkg_info(stack_recommendation, unknown_pkgs)
if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW") != "1" and ecosystem != "golang":
# Unknown Packages is Present and INGESTION is Enabled
logger.debug(unknown_pkgs)
unknown_package_flow(ecosystem, unknown_pkgs)
return response_template(stack_recommendation, 202, headers)
return response_template(stack_recommendation, 200, headers)
def test_get_batch_ca_data_empty(self):
"""Test Ca batch data."""
result = get_batch_ca_data('golang', [])
self.assertIsInstance(result, dict)
self.assertEqual(result, {})
