def call_pgm(self, payload):
"""Call the PGM model.
Calls the PGM model with the normalized manifest information to get
the relevant packages.
"""
try:
# TODO remove hardcodedness for payloads with multiple ecosystems
if payload and 'ecosystem' in payload[0]:
PGM_SERVICE_HOST = os.environ.get(
"PGM_SERVICE_HOST") + "-" + payload[0]['ecosystem']
PGM_URL_REST = "http://{host}:{port}".format(
host=PGM_SERVICE_HOST,
port=os.environ.get("PGM_SERVICE_PORT"))
pgm_url = PGM_URL_REST + "/api/v1/schemas/kronos_scoring"
response = get_session_retry().post(pgm_url, json=payload)
if response.status_code != 200:
self.log.error("HTTP error {}. Error retrieving PGM data.".format(
response.status_code))
return None
else:
json_response = response.json()
return json_response
else:
self.log.debug('Payload information is not passed in the call, '
'Quitting! PGM\'s call')
except Exception:
self.log.error("Failed retrieving PGM data.")
return None
def _get_dependency_data(self, resolved, ecosystem):
# Hardcoded ecosystem
result = []
for elem in resolved:
qstring = ("g.V().has('pecosystem','" + ecosystem +
"').has('pname','" + elem["package"] +
"').has('version','" + elem["version"] + "').")
qstring += ("as('version').in('has_version').as('package')." +
"select('version','package').by(valueMap());")
payload = {'gremlin': qstring}
try:
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
if graph_resp.get('result', {}).get('data'):
result.append(graph_resp["result"])
else:
self.log.error("Failed retrieving dependency data.")
continue
except Exception:
self.log.exception("Error retrieving dependency data.")
continue
return {"result": result}
def get_dependency_data(dependency_list):
ecosystem = "maven"
dep_pkg_list_unknown = []
dep_pkg_list_known = []
for item in dependency_list:
dependency_list = item.split(":")
result = []
name = dependency_list[0] + ":" + dependency_list[1]
version = dependency_list[2]
qstring = ("g.V().has('pecosystem','" + ecosystem +
"').has('pname','" + name + "').has('version','" + version +
"').tryNext()")
payload = {'gremlin': qstring}
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
if graph_resp.get('result', {}).get('data'):
result.append(graph_resp["result"])
if result[0]['data'][0]['present']:
dep_pkg_list_known.append(ecosystem + ":" + name + ":" +
version)
elif not (result[0]['data'][0]['present']):
dep_pkg_list_unknown.append(ecosystem + ":" + name + ":" +
version)
else:
continue
else:
continue
print()
return dep_pkg_list_unknown
def get_dependency_data(self, dependency_list):
"""Prepare list of unknown dependencies from given list of dependencies."""
ecosystem = "maven"
dep_pkg_list_unknown = []
# TODO: do we need this list? it is filled in the code and then the results are forgotten
dep_pkg_list_known = []
for item in dependency_list:
dependency_list = item.split(":")
result = []
name = dependency_list[0] + ":" + dependency_list[1]
version = dependency_list[2]
qstring = ("g.V().has('pecosystem','" + ecosystem +
"').has('pname','" + name + "').has('version','" +
version + "').tryNext()")
payload = {'gremlin': qstring}
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
if graph_resp.get('result', {}).get('data'):
result.append(graph_resp["result"])
if result[0]['data'][0]['present']:
dep_pkg_list_known.append(ecosystem + ":" + name +
":" + version)
elif not (result[0]['data'][0]['present']):
dep_pkg_list_unknown.append(ecosystem + ":" + name +
":" + version)
else:
continue
else:
continue
return dep_pkg_list_unknown
def get_dependency_data(resolved, ecosystem):
result = []
for elem in resolved:
if elem["package"] is None or elem["version"] is None:
_logger.warning("Either component name or component version is missing")
continue
qstring = \
"g.V().has('pecosystem', '{}').has('pname', '{}').has('version', '{}')" \
.format(ecosystem, elem["package"], elem["version"]) + \
".as('version').in('has_version').as('package')" + \
".select('version','package').by(valueMap());"
payload = {'gremlin': qstring}
try:
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
if 'result' not in graph_resp:
continue
if len(graph_resp['result']['data']) == 0:
continue
result.append(graph_resp["result"])
else:
_logger.error("Failed retrieving dependency data.")
continue
except Exception:
_logger.exception("Error retrieving dependency data!")
continue
return {"result": result}
def get_dependency_data(self, dependency_list):
"""Prepare list of unknown dependencies from given list of dependencies."""
ecosystem = "maven"
dep_pkg_list_unknown = list()
dep_pkg_list_known = list()
for dependency in dependency_list:
dependency_list = dependency.split(":")
name = dependency_list[0] + ":" + dependency_list[1]
version = dependency_list[2]
qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" +
name + "').has('version','" + version + "').tryNext()")
payload = {'gremlin': qstring}
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp_data = graph_req.json().get('result', {}).get('data')
if graph_resp_data[0].get('present'):
dep_pkg_list_known.append(ecosystem + ":" + name + ":" + version)
else:
dep_pkg_list_unknown.append(ecosystem + ":" + name + ":" + version)
else:
self.log.error("Error response from graph for {dependency} " +
"with status code as {status_code}"
.format(dependency=dependency, status_code=graph_req.status_code))
self.log.debug("Known dependencies are: {}".format(dep_pkg_list_known))
self.log.debug("Unknown dependencies are: {}".format(dep_pkg_list_unknown))
return dep_pkg_list_unknown
def _execute_query(self, query):
"""Run the graph query."""
payload = {'gremlin': query}
graph_url = self._get_graph_url()
self.log.debug(query)
response = get_session_retry().post(graph_url,
data=json.dumps(payload))
if response.status_code == 200:
return response.json()
else:
self.log.error('Graph is not responding.')
return {}
def get_osio_user_count(ecosystem, name, version):
str_gremlin = "g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "')." \
"has('version','" + version + "').in('uses').count();"
payload = {'gremlin': str_gremlin}
try:
response = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
json_response = response.json()
return json_response['result']['data'][0]
except Exception:
logger.error("Failed retrieving Gremlin data.")
return -1
def store_user_node(self, arguments, aggregated):
"""Store GraphAggregatorTask's result to graph."""
for result in aggregated['result']:
resolved = result['details'][0]['_resolved']
ecosystem = result['details'][0]['ecosystem']
email = arguments.get('data').get('user_profile').get('email')
company = arguments.get('data').get('user_profile').get(
'company', 'Not Provided')
# Create User Node if it does not exist
qstring = "user = g.V().has('userid','" + email + \
"').tryNext().orElseGet{graph.addVertex(" \
"'vertex_label','User','userid','" + email + "', 'company','" + company + \
"')}; g.V(user).as('u')"
for epvs in resolved:
if epvs['package'] is None or epvs['version'] is None:
self.log.warning(
"Either component name or component version is missing."
)
continue
# Create Version Node if it does not exist
qstring += ".coalesce(g.V().has('pecosystem','" + ecosystem + "')." + \
"has('pname','" + epvs['package'] + "')." + \
"has('version','" + epvs['version'] + \
"'), addV().property('vertex_label','Version', " + \
"'pecosystem','" + ecosystem + "','pname', '" + epvs['package'] + \
"', " \
"'version', '" + epvs['version'] + "')).as('ver_" + epvs['package'] + \
"')"
# Check if "user -> uses -> version" edge exists else create one
qstring += ".coalesce(inE('uses').where(outV().as('u')), " \
"addE('uses').from('u').to('ver_" + epvs['package'] + "')" \
".coalesce(inV().has('osio_usage_count').sack(assign)." \
"by('osio_usage_count')." \
"sack(sum).by(constant(1))." \
"property('osio_usage_count', sack()), inV()." \
"property('osio_usage_count', 1)))"
payload = {'gremlin': qstring}
try:
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code != 200:
self.log.error(
"Failed creating book-keeping record in graph")
continue
except Exception:
self.log.exception("Failed to communicate to Graph Server.")
continue
def execute_gremlin_dsl(self, payload):
"""Execute gremlin query and return the response."""
try:
response = get_session_retry().post(self._bayesian_graph_url, data=json.dumps(payload))
if response.status_code != 200:
_logger.error("HTTP error {}. Error retrieving Gremlin data.".format(
response.status_code))
return None
else:
json_response = response.json()
return json_response
except Exception:
_logger.error("Failed retrieving Gremlin data.")
return None
def _get_dependency_data(self, dependencies):
dependency_data_list = list()
self.log.debug("Dependencies are: {}".format(dependencies))
for dependency in dependencies:
self.log.info("Analyzing dependency: {}".format(dependency))
n_colons = dependency.count(":")
dependency_list = dependency.split(":")
ecosystem = dependency_list[0]
version = dependency_list[-1]
if n_colons == 3:
name = dependency_list[1] + ":" + dependency_list[2]
elif n_colons == 2:
name = dependency_list[1]
else:
self.log.error(
"No valid dependency format found: {}".format(dependency))
name = ""
qstring = ("g.V().has('pecosystem','" + ecosystem +
"').has('pname','" + name + "')"
".has('version','" + version + "').")
qstring += (
"as('version').in('has_version').as('package').dedup()." +
"select('version','package').by(valueMap());")
payload = {'gremlin': qstring}
try:
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
data = graph_resp.get('result', {}).get('data')
if data:
version_data = self.parse_version_data(
data[0].get('version'))
package_data = self.parse_package_data(
data[0].get('package'))
dependency_data = version_data.copy()
dependency_data.update(package_data)
dependency_data_list.append(dependency_data)
else:
self.log.error("Failed retrieving dependency data.")
continue
except Exception:
self.log.exception("Error retrieving dependency data.")
continue
self.log.debug(
"Dependency data list is: {}".format(dependency_data_list))
return dependency_data_list
def perform_license_analysis(license_score_list, dependencies):
"""Pass given license_score_list to stack_license analysis and process response."""
license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license"
payload = {"packages": license_score_list}
resp = {}
flag_stack_license_exception = False
try:
lic_response = get_session_retry().post(license_url,
data=json.dumps(payload))
lic_response.raise_for_status(
) # raise exception for bad http-status codes
resp = lic_response.json()
except requests.exceptions.RequestException:
_logger.exception(
"Unexpected error happened while invoking license analysis!")
flag_stack_license_exception = True
pass
stack_license = []
stack_license_status = None
unknown_licenses = []
license_conflict_packages = []
license_outliers = []
if not flag_stack_license_exception:
list_components = resp.get('packages', [])
for comp in list_components: # output from license analysis
for dep in dependencies: # the known dependencies
if dep.get('name', '') == comp.get('package', '') and \
dep.get('version', '') == comp.get('version', ''):
dep['license_analysis'] = comp.get('license_analysis', {})
_stack_license = resp.get('stack_license', None)
if _stack_license is not None:
stack_license = [_stack_license]
stack_license_status = resp.get('status', None)
unknown_licenses = _extract_unknown_licenses(resp)
license_conflict_packages = _extract_conflict_packages(resp)
license_outliers = _extract_license_outliers(resp)
output = {
"status": stack_license_status,
"f8a_stack_licenses": stack_license,
"unknown_licenses": unknown_licenses,
"conflict_packages": license_conflict_packages,
"outlier_packages": license_outliers
}
return output, dependencies
def invoke_license_analysis_service(user_stack_packages, alternate_packages, companion_packages):
"""Pass given args to stack_license analysis."""
license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license"
payload = {
"packages": user_stack_packages,
"alternate_packages": alternate_packages,
"companion_packages": companion_packages
}
json_response = {}
try:
lic_response = get_session_retry().post(license_url, data=json.dumps(payload))
lic_response.raise_for_status() # raise exception for bad http-status codes
json_response = lic_response.json()
except requests.exceptions.RequestException:
_logger.exception("Unexpected error happened while invoking license analysis!")
pass
return json_response
def _get_dependency_data(self, dependencies, ecosystem):
dependency_data_list = list()
self.log.debug("Dependencies are: {}".format(dependencies))
for dependency in dependencies:
self.log.info("Analyzing dependency: {}".format(dependency))
artifact_coords = MavenCoordinates.from_str(dependency)
qstring = ("g.V().has('pecosystem','" + ecosystem +
"').has('pname','" + artifact_coords.groupId + ":" +
artifact_coords.artifactId + "')"
".has('version','" + artifact_coords.version + "').")
qstring += (
"as('version').in('has_version').as('package').dedup()." +
"select('version','package').by(valueMap());")
payload = {'gremlin': qstring}
try:
graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST,
data=json.dumps(payload))
if graph_req.status_code == 200:
graph_resp = graph_req.json()
data = graph_resp.get('result', {}).get('data')
if data:
version_data = self.parse_version_data(
data[0].get('version'))
package_data = self.parse_package_data(
data[0].get('package'))
dependency_data = version_data.copy()
dependency_data.update(package_data)
dependency_data_list.append(dependency_data)
else:
self.log.error("Failed retrieving dependency data.")
continue
except Exception:
self.log.exception("Error retrieving dependency data.")
continue
self.log.debug(
"Dependency data list is: {}".format(dependency_data_list))
return dependency_data_list
