def call_pgm(self, payload): """Call the PGM model. Calls the PGM model with the normalized manifest information to get the relevant packages. """ try: # TODO remove hardcodedness for payloads with multiple ecosystems if payload and 'ecosystem' in payload[0]: PGM_SERVICE_HOST = os.environ.get( "PGM_SERVICE_HOST") + "-" + payload[0]['ecosystem'] PGM_URL_REST = "http://{host}:{port}".format( host=PGM_SERVICE_HOST, port=os.environ.get("PGM_SERVICE_PORT")) pgm_url = PGM_URL_REST + "/api/v1/schemas/kronos_scoring" response = get_session_retry().post(pgm_url, json=payload) if response.status_code != 200: self.log.error("HTTP error {}. Error retrieving PGM data.".format( response.status_code)) return None else: json_response = response.json() return json_response else: self.log.debug('Payload information is not passed in the call, ' 'Quitting! PGM\'s call') except Exception: self.log.error("Failed retrieving PGM data.") return None
def _get_dependency_data(self, resolved, ecosystem): # Hardcoded ecosystem result = [] for elem in resolved: qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + elem["package"] + "').has('version','" + elem["version"] + "').") qstring += ("as('version').in('has_version').as('package')." + "select('version','package').by(valueMap());") payload = {'gremlin': qstring} try: graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() if graph_resp.get('result', {}).get('data'): result.append(graph_resp["result"]) else: self.log.error("Failed retrieving dependency data.") continue except Exception: self.log.exception("Error retrieving dependency data.") continue return {"result": result}
def get_dependency_data(dependency_list): ecosystem = "maven" dep_pkg_list_unknown = [] dep_pkg_list_known = [] for item in dependency_list: dependency_list = item.split(":") result = [] name = dependency_list[0] + ":" + dependency_list[1] version = dependency_list[2] qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "').has('version','" + version + "').tryNext()") payload = {'gremlin': qstring} graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() if graph_resp.get('result', {}).get('data'): result.append(graph_resp["result"]) if result[0]['data'][0]['present']: dep_pkg_list_known.append(ecosystem + ":" + name + ":" + version) elif not (result[0]['data'][0]['present']): dep_pkg_list_unknown.append(ecosystem + ":" + name + ":" + version) else: continue else: continue print() return dep_pkg_list_unknown
def get_dependency_data(self, dependency_list): """Prepare list of unknown dependencies from given list of dependencies.""" ecosystem = "maven" dep_pkg_list_unknown = [] # TODO: do we need this list? it is filled in the code and then the results are forgotten dep_pkg_list_known = [] for item in dependency_list: dependency_list = item.split(":") result = [] name = dependency_list[0] + ":" + dependency_list[1] version = dependency_list[2] qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "').has('version','" + version + "').tryNext()") payload = {'gremlin': qstring} graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() if graph_resp.get('result', {}).get('data'): result.append(graph_resp["result"]) if result[0]['data'][0]['present']: dep_pkg_list_known.append(ecosystem + ":" + name + ":" + version) elif not (result[0]['data'][0]['present']): dep_pkg_list_unknown.append(ecosystem + ":" + name + ":" + version) else: continue else: continue return dep_pkg_list_unknown
def get_dependency_data(resolved, ecosystem): result = [] for elem in resolved: if elem["package"] is None or elem["version"] is None: _logger.warning("Either component name or component version is missing") continue qstring = \ "g.V().has('pecosystem', '{}').has('pname', '{}').has('version', '{}')" \ .format(ecosystem, elem["package"], elem["version"]) + \ ".as('version').in('has_version').as('package')" + \ ".select('version','package').by(valueMap());" payload = {'gremlin': qstring} try: graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() if 'result' not in graph_resp: continue if len(graph_resp['result']['data']) == 0: continue result.append(graph_resp["result"]) else: _logger.error("Failed retrieving dependency data.") continue except Exception: _logger.exception("Error retrieving dependency data!") continue return {"result": result}
def get_dependency_data(self, dependency_list): """Prepare list of unknown dependencies from given list of dependencies.""" ecosystem = "maven" dep_pkg_list_unknown = list() dep_pkg_list_known = list() for dependency in dependency_list: dependency_list = dependency.split(":") name = dependency_list[0] + ":" + dependency_list[1] version = dependency_list[2] qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "').has('version','" + version + "').tryNext()") payload = {'gremlin': qstring} graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp_data = graph_req.json().get('result', {}).get('data') if graph_resp_data[0].get('present'): dep_pkg_list_known.append(ecosystem + ":" + name + ":" + version) else: dep_pkg_list_unknown.append(ecosystem + ":" + name + ":" + version) else: self.log.error("Error response from graph for {dependency} " + "with status code as {status_code}" .format(dependency=dependency, status_code=graph_req.status_code)) self.log.debug("Known dependencies are: {}".format(dep_pkg_list_known)) self.log.debug("Unknown dependencies are: {}".format(dep_pkg_list_unknown)) return dep_pkg_list_unknown
def _execute_query(self, query): """Run the graph query.""" payload = {'gremlin': query} graph_url = self._get_graph_url() self.log.debug(query) response = get_session_retry().post(graph_url, data=json.dumps(payload)) if response.status_code == 200: return response.json() else: self.log.error('Graph is not responding.') return {}
def get_osio_user_count(ecosystem, name, version): str_gremlin = "g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "')." \ "has('version','" + version + "').in('uses').count();" payload = {'gremlin': str_gremlin} try: response = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) json_response = response.json() return json_response['result']['data'][0] except Exception: logger.error("Failed retrieving Gremlin data.") return -1
def store_user_node(self, arguments, aggregated): """Store GraphAggregatorTask's result to graph.""" for result in aggregated['result']: resolved = result['details'][0]['_resolved'] ecosystem = result['details'][0]['ecosystem'] email = arguments.get('data').get('user_profile').get('email') company = arguments.get('data').get('user_profile').get( 'company', 'Not Provided') # Create User Node if it does not exist qstring = "user = g.V().has('userid','" + email + \ "').tryNext().orElseGet{graph.addVertex(" \ "'vertex_label','User','userid','" + email + "', 'company','" + company + \ "')}; g.V(user).as('u')" for epvs in resolved: if epvs['package'] is None or epvs['version'] is None: self.log.warning( "Either component name or component version is missing." ) continue # Create Version Node if it does not exist qstring += ".coalesce(g.V().has('pecosystem','" + ecosystem + "')." + \ "has('pname','" + epvs['package'] + "')." + \ "has('version','" + epvs['version'] + \ "'), addV().property('vertex_label','Version', " + \ "'pecosystem','" + ecosystem + "','pname', '" + epvs['package'] + \ "', " \ "'version', '" + epvs['version'] + "')).as('ver_" + epvs['package'] + \ "')" # Check if "user -> uses -> version" edge exists else create one qstring += ".coalesce(inE('uses').where(outV().as('u')), " \ "addE('uses').from('u').to('ver_" + epvs['package'] + "')" \ ".coalesce(inV().has('osio_usage_count').sack(assign)." \ "by('osio_usage_count')." \ "sack(sum).by(constant(1))." \ "property('osio_usage_count', sack()), inV()." \ "property('osio_usage_count', 1)))" payload = {'gremlin': qstring} try: graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code != 200: self.log.error( "Failed creating book-keeping record in graph") continue except Exception: self.log.exception("Failed to communicate to Graph Server.") continue
def execute_gremlin_dsl(self, payload): """Execute gremlin query and return the response.""" try: response = get_session_retry().post(self._bayesian_graph_url, data=json.dumps(payload)) if response.status_code != 200: _logger.error("HTTP error {}. Error retrieving Gremlin data.".format( response.status_code)) return None else: json_response = response.json() return json_response except Exception: _logger.error("Failed retrieving Gremlin data.") return None
def _get_dependency_data(self, dependencies): dependency_data_list = list() self.log.debug("Dependencies are: {}".format(dependencies)) for dependency in dependencies: self.log.info("Analyzing dependency: {}".format(dependency)) n_colons = dependency.count(":") dependency_list = dependency.split(":") ecosystem = dependency_list[0] version = dependency_list[-1] if n_colons == 3: name = dependency_list[1] + ":" + dependency_list[2] elif n_colons == 2: name = dependency_list[1] else: self.log.error( "No valid dependency format found: {}".format(dependency)) name = "" qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + name + "')" ".has('version','" + version + "').") qstring += ( "as('version').in('has_version').as('package').dedup()." + "select('version','package').by(valueMap());") payload = {'gremlin': qstring} try: graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() data = graph_resp.get('result', {}).get('data') if data: version_data = self.parse_version_data( data[0].get('version')) package_data = self.parse_package_data( data[0].get('package')) dependency_data = version_data.copy() dependency_data.update(package_data) dependency_data_list.append(dependency_data) else: self.log.error("Failed retrieving dependency data.") continue except Exception: self.log.exception("Error retrieving dependency data.") continue self.log.debug( "Dependency data list is: {}".format(dependency_data_list)) return dependency_data_list
def perform_license_analysis(license_score_list, dependencies): """Pass given license_score_list to stack_license analysis and process response.""" license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" payload = {"packages": license_score_list} resp = {} flag_stack_license_exception = False try: lic_response = get_session_retry().post(license_url, data=json.dumps(payload)) lic_response.raise_for_status( ) # raise exception for bad http-status codes resp = lic_response.json() except requests.exceptions.RequestException: _logger.exception( "Unexpected error happened while invoking license analysis!") flag_stack_license_exception = True pass stack_license = [] stack_license_status = None unknown_licenses = [] license_conflict_packages = [] license_outliers = [] if not flag_stack_license_exception: list_components = resp.get('packages', []) for comp in list_components: # output from license analysis for dep in dependencies: # the known dependencies if dep.get('name', '') == comp.get('package', '') and \ dep.get('version', '') == comp.get('version', ''): dep['license_analysis'] = comp.get('license_analysis', {}) _stack_license = resp.get('stack_license', None) if _stack_license is not None: stack_license = [_stack_license] stack_license_status = resp.get('status', None) unknown_licenses = _extract_unknown_licenses(resp) license_conflict_packages = _extract_conflict_packages(resp) license_outliers = _extract_license_outliers(resp) output = { "status": stack_license_status, "f8a_stack_licenses": stack_license, "unknown_licenses": unknown_licenses, "conflict_packages": license_conflict_packages, "outlier_packages": license_outliers } return output, dependencies
def invoke_license_analysis_service(user_stack_packages, alternate_packages, companion_packages): """Pass given args to stack_license analysis.""" license_url = LICENSE_SCORING_URL_REST + "/api/v1/stack_license" payload = { "packages": user_stack_packages, "alternate_packages": alternate_packages, "companion_packages": companion_packages } json_response = {} try: lic_response = get_session_retry().post(license_url, data=json.dumps(payload)) lic_response.raise_for_status() # raise exception for bad http-status codes json_response = lic_response.json() except requests.exceptions.RequestException: _logger.exception("Unexpected error happened while invoking license analysis!") pass return json_response
def _get_dependency_data(self, dependencies, ecosystem): dependency_data_list = list() self.log.debug("Dependencies are: {}".format(dependencies)) for dependency in dependencies: self.log.info("Analyzing dependency: {}".format(dependency)) artifact_coords = MavenCoordinates.from_str(dependency) qstring = ("g.V().has('pecosystem','" + ecosystem + "').has('pname','" + artifact_coords.groupId + ":" + artifact_coords.artifactId + "')" ".has('version','" + artifact_coords.version + "').") qstring += ( "as('version').in('has_version').as('package').dedup()." + "select('version','package').by(valueMap());") payload = {'gremlin': qstring} try: graph_req = get_session_retry().post(GREMLIN_SERVER_URL_REST, data=json.dumps(payload)) if graph_req.status_code == 200: graph_resp = graph_req.json() data = graph_resp.get('result', {}).get('data') if data: version_data = self.parse_version_data( data[0].get('version')) package_data = self.parse_package_data( data[0].get('package')) dependency_data = version_data.copy() dependency_data.update(package_data) dependency_data_list.append(dependency_data) else: self.log.error("Failed retrieving dependency data.") continue except Exception: self.log.exception("Error retrieving dependency data.") continue self.log.debug( "Dependency data list is: {}".format(dependency_data_list)) return dependency_data_list