def get_dependency_data(epv_set):
"""Get dependency data from graph."""
epv_list = {"result": {"data": [], "unknown_deps": []}}
unknown_deps_list = []
query = "epv=[];"
batch_query = "a = g.V().has('pecosystem', '{eco}').has('pname', '{name}')." \
"has('version', '{ver}').dedup(); a.clone().as('version')." \
"in('has_version').dedup().as('package').select('version')." \
"coalesce(out('has_cve').as('cve')." \
"select('package','version','cve').by(valueMap())," \
"select('package','version').by(valueMap()))." \
"fill(epv);"
i = 1
epvs = [x for x, y in epv_set['direct'].items()]
dep_list = []
for epv in epvs:
eco, name, ver = epv.split('|#|')
dep_list.append((name, ver))
query += batch_query.format(eco=eco, name=name, ver=ver)
if i >= GREMLIN_QUERY_SIZE:
i = 1
# call_gremlin in batch
payload = {'gremlin': query}
result = post_http_request(url=GREMLIN_SERVER_URL_REST,
payload=payload)
if result:
epv_list['result']['data'] += result['result']['data']
query = "epv=[];"
i += 1
if i > 1:
payload = {'gremlin': query}
result = post_http_request(url=GREMLIN_SERVER_URL_REST,
payload=payload)
if result:
epv_list['result']['data'] += result['result']['data']
tr_epv_list, tr_list = get_tr_dependency_data(epv_set)
transitive_count = len(tr_epv_list['result']['data'])
# Identification of unknown direct dependencies
epv_data = epv_list['result']['data']
epv_list, unknown_deps_list = find_unknown_deps(epv_data, epv_list,
dep_list,
unknown_deps_list)
# Identification of unknown transitive dependencies
epv_data = tr_epv_list['result']['data']
epv_list, unknown_deps_list = find_unknown_deps(epv_data, epv_list,
tr_list, unknown_deps_list,
True)
result = add_transitive_details(epv_list, epv_set)
accumulated_data = {
'result': result,
'unknown_deps': unknown_deps_list,
'transitive_count': transitive_count
}
logger.info('Accumulated data: {}'.format(accumulated_data))
return accumulated_data
def get_version_information(input_list, ecosystem):
"""Fetch the version information for each of the packages.
Also remove EPVs with CVEs and ones not present in Graph
"""
str_query = "data=[]; "
for package in input_list:
str_query += "pkg = g.V().has('ecosystem', '{eco}').has('name', '{pkg}'); " \
"lnv = []; pkg.clone().values('latest_non_cve_version', " \
"'latest_version').fill(lnv); pkg.clone().as('package').V()." \
"has('pecosystem', '{eco}').has('pname', '{pkg}')." \
"has('version', within(lnv)).as('version')." \
"select('package', 'version').by(valueMap()).fill(data);".format(
eco=ecosystem,
pkg=package)
str_query += "data"
payload = {'gremlin': str_query}
# Query Gremlin with packages list to get their version information
gremlin_response = post_http_request(url=GREMLIN_SERVER_URL_REST,
payload=payload)
if gremlin_response is None:
return []
response = get_response_data(gremlin_response, [{0: 0}])
return response
def get_license_analysis_for_stack(
package_details: List[PackageDetails]) -> LicenseAnalysis: # pylint:disable=R0914
"""Create LicenseAnalysis from license server."""
license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license"
# form payload for license service request
payload = {
"packages": get_license_service_request_payload(package_details)
}
# (fixme) refactoring
try:
resp = post_http_request(url=license_url, payload=payload)
except Exception as e:
logger.exception("Unexpected error(%s) happened while invoking license analysis!",
e)
else:
unknown_licenses = _extract_unknown_licenses(resp)
license_conflict_packages = _extract_conflict_packages(resp)
license_outliers = _extract_license_outliers(resp)
stack_license = resp.get('stack_license', None)
stack_license = [stack_license] if stack_license else None
reason = resp.get('message')
status = resp.get('status', None)
stack_distinct_licenses = list(get_distinct_licenses(package_details))
return LicenseAnalysis(reason=reason, status=status,
recommended_licenses=stack_license,
distinct_licenses=stack_distinct_licenses,
unknown_licenses=unknown_licenses,
conflict_packages=license_conflict_packages,
outlier_packages=license_outliers)
return LicenseAnalysis()
def get_tr_dependency_data(epv_set):
"""Get transitive dependency data from graph."""
query = "epv=[];"
tr_epv_list = {
"result": {
"data": []
}
}
batch_query = "a = g.V().has('pecosystem', '{eco}').has('pname', '{name}')." \
"has('version', '{ver}').dedup(); a.clone().as('version')." \
"in('has_version').dedup().as('package').select('version')." \
"coalesce(out('has_cve').as('cve')." \
"select('package','version','cve').by(valueMap())," \
"select('package','version').by(valueMap()))." \
"fill(epv);"
i = 1
epvs = [x for x, y in epv_set['transitive'].items()]
tr_list = []
for epv in epvs:
eco, name, ver = epv.split('|#|')
tr_list.append((name, ver))
query += batch_query.format(eco=eco, name=name, ver=ver)
if i >= GREMLIN_QUERY_SIZE:
i = 1
# call_gremlin in batch
payload = {'gremlin': query}
result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
if result:
tr_epv_list['result']['data'] += result['result']['data']
query = "epv=[];"
i += 1
if i > 1:
payload = {'gremlin': query}
time_start = time.time()
result = post_http_request(url=SETTINGS.gremlin_url, payload=payload)
logger.info('elapsed_time for gremlin call: {}'.format(time.time() - time_start))
if result:
tr_epv_list['result']['data'] += result['result']['data']
return tr_epv_list, tr_list
def perform_license_analysis(license_score_list, dependencies):
"""Pass given license_score_list to stack_license analysis and process response."""
license_url = AGGREGATOR_SETTINGS.license_analysis_base_url + "/api/v1/stack_license"
payload = {
"packages": license_score_list
}
resp = {}
flag_stack_license_exception = False
# TODO: refactoring
try:
resp = post_http_request(url=license_url, payload=payload)
# lic_response.raise_for_status() # raise exception for bad http-status codes
if not resp:
raise requests.exceptions.RequestException
except requests.exceptions.RequestException:
current_app.logger.exception("Unexpected error happened while invoking license analysis!")
flag_stack_license_exception = True
msg = None
stack_license = []
stack_license_status = None
unknown_licenses = []
license_conflict_packages = []
license_outliers = []
if not flag_stack_license_exception:
list_components = resp.get('packages', [])
for comp in list_components: # output from license analysis
for dep in dependencies: # the known dependencies
if dep.get('name', '') == comp.get('package', '') and \
dep.get('version', '') == comp.get('version', ''):
dep['license_analysis'] = comp.get('license_analysis', {})
msg = resp.get('message')
_stack_license = resp.get('stack_license', None)
if _stack_license is not None:
stack_license = [_stack_license]
stack_license_status = resp.get('status', None)
unknown_licenses = _extract_unknown_licenses(resp)
license_conflict_packages = _extract_conflict_packages(resp)
license_outliers = _extract_license_outliers(resp)
output = {
"reason": msg,
"status": stack_license_status,
"f8a_stack_licenses": stack_license,
"unknown_licenses": unknown_licenses,
"conflict_packages": license_conflict_packages,
"outlier_packages": license_outliers
}
return output, dependencies
def get_recommended_version(ecosystem, name, version):
"""Fetch the recommended version in case of CVEs."""
query = "g.V().has('ecosystem', '{eco}').has('name', '{pkg}')" \
".out('has_version').not(out('has_cve')).values('version');"\
.format(eco=ecosystem, pkg=name)
payload = {'gremlin': query}
result = post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
if result:
versions = result['result']['data']
if len(versions) == 0:
return None
else:
return None
rec_version = version
for ver in versions:
rec_version = select_latest_version(ver, rec_version)
if rec_version == version:
return None
return rec_version
def test_post_http_request(_mock1):
"""Test error response for gremlin."""
payload = {'gremlin': ''}
with raises(RequestException):
post_http_request(url=SETTINGS.gremlin_url, payload=payload)
def test_post_http_request(_mock1):
"""Test error response for gremlin."""
payload = {'gremlin': ''}
with raises(RequestException):
post_http_request(url=GREMLIN_SERVER_URL_REST, payload=payload)
