def test_construct_version_query_4(): """Test the GraphPopulator.construct_version_query() class method.""" input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": { "details": [{ "description": "Some description here", "declared_license": "GPL and\nv2.0" }] } } } q = GraphPopulator.construct_version_query(input_json) logger.info(q) assert "access_points" in q assert "0.4.59" in q assert "pypi" in q input_json = { "version": "deb579d6e030503f430978ee229008b9bc912d40", "package": "github.com/gorilla/mux", "ecosystem": "go", "analyses": { "source_licenses": { "status": "success", "summary": { "sure_licenses": ["BSD-Modified"] } }, "metadata": { "details": [{ "code_repository": { "type": "git", "url": "https://github.com/gorilla/mux" }, "dependencies": [], "ecosystem": "gofedlib", "name": "github.com/gorilla/mux", "version": "deb579d6e030503f430978ee229008b9bc912d40" }] } } } q = GraphPopulator.construct_version_query(input_json) assert "'declared_licenses'" in q assert "'licenses'" in q assert "BSD-Modified" in q
def create_pv_nodes(self): """Create Package and Version nodes, if needed.""" nodes = [ ] # return (e, p, v) tuples of created/existing nodes; for easier testing affected_pkgs = {} all_epvs_created = True for pv_dict in self._cve_dict.get('affected'): epv_dict = pv_dict.copy() epv_dict['ecosystem'] = self._cve_dict.get('ecosystem') query = GraphPopulator.construct_graph_nodes(epv_dict) latest_version = "-1" # Fetch the value of the latest_version from the query created if "latest_version" in query: data = query.split("\'latest_version\'")[1].split(");")[0] latest_version = data.replace(",", "").strip().replace("'", "") success, json_response = BayesianGraph.execute(query) e = epv_dict.get('ecosystem') p = epv_dict.get('name') v = epv_dict.get('version') if p not in affected_pkgs: tmp = {"ecosystem": e, "latest_version": latest_version} affected_pkgs[p] = tmp if not success: logger.error( 'CVEIngestionError - Error creating nodes for {e}/{p}/{v}: {r}' .format(e=e, p=p, v=v, r=str(json_response))) all_epvs_created = False else: nodes.append((e, p, v)) return nodes, all_epvs_created, affected_pkgs
def create_pv_nodes(self): """Create Package and Version nodes, if needed.""" nodes = [] # return (e, p, v) tuples of created/existing nodes; for easier testing affected_pkgs = {} all_epvs_created = True for pv_dict in self._cve_dict.get('affected'): epv_dict = pv_dict.copy() epv_dict['ecosystem'] = self._cve_dict.get('ecosystem') query, bindings = GraphPopulator.construct_graph_nodes(epv_dict) # Fetch the value of the latest_version from the query created latest_version = -1 if "latest" in bindings: latest_version = bindings['latest'] success, json_response = BayesianGraph.execute(self.prepare_payload(query, bindings)) e = epv_dict.get('ecosystem') p = epv_dict.get('name') v = epv_dict.get('version') if p not in affected_pkgs: tmp = { "ecosystem": e, "latest_version": latest_version } affected_pkgs[p] = tmp if not success: logger.error('CVEIngestionError - Error creating nodes for {e}/{p}/{v}: {r}'.format( e=e, p=p, v=v, r=str(json_response)) ) all_epvs_created = False else: nodes.append((e, p, v)) return nodes, all_epvs_created, affected_pkgs
def create_pv_nodes(self): """Create Package and Version nodes, if needed.""" nodes = [ ] # return (e, p, v) tuples of created/existing nodes; for easier testing affected_pkgs = {} all_epvs_created = True p = self._cve_dict.get('package') e = self._cve_dict.get('ecosystem') epv_dict = { "ecosystem": self._cve_dict.get('ecosystem'), "name": self._cve_dict.get('package') } latest_version = "" for ver in self._cve_dict.get('affected'): epv_dict['version'] = ver query = GraphPopulator.construct_graph_nodes(epv_dict) success, json_response = BayesianGraph.execute(query) # Fetch the value of the latest_version from the query create if not latest_version and "latest_version" in query: data = query.split("\'latest_version\'")[1].split(");")[0] latest_version = data.replace(",", "").strip().replace("'", "") if not success: logger.error( 'CVEIngestionError - Error creating nodes for {e}/{p}/{v}: {r}' .format(e=e, p=p, v=ver, r=str(json_response))) all_epvs_created = False else: nodes.append((e, p, ver)) if p not in affected_pkgs: affected_pkg = {"ecosystem": e, "latest_version": latest_version} affected_pkgs[p] = affected_pkg # To create the latest version node if not present if latest_version and latest_version != "-1": epv_dict['version'] = latest_version query = GraphPopulator.construct_graph_nodes(epv_dict) BayesianGraph.execute(query) return nodes, all_epvs_created, affected_pkgs
def test_construct_version_query_2(): """Test the GraphPopulator.construct_version_query() class method.""" input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": { "details": [{ "description": "Some description here", "declared_licenses": ["GPL v3", "APL v2.0"] }] }, "github_details": {}, 'libraries_io': {}, 'source_licenses': {}, 'security_issues': { "details": [{ "id": "CEV-007", "cvss": { "score": 9.7 } }] }, 'code_metrics': { "details": { "languages": [{ "metrics": { "functions": { 'average_cyclomatic_complexity': 3 } } }] } }, 'redhat_downstream': { "summary": { "all_rhsm_product_names": ["access_points_rh"] } } } } q = GraphPopulator.construct_version_query(input_json) logger.info(q) assert "access_points" in q assert "0.4.59" in q assert "pypi" in q
def test_construct_version_query_1(): """Test the GraphPopulator.construct_version_query() class method.""" input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": {} } } q = GraphPopulator.construct_version_query(input_json) logger.info(q) assert "access_points" in q assert "0.4.59" in q assert "pypi" in q assert "addVertex" in q assert "drop()" not in q
def test_construct_version_query_3(): """Test the GraphPopulator.construct_version_query() class method.""" input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": {"details": [ {"description": "Some description here", "declared_license": "GPL \nv2.0"} ]} } } q = GraphPopulator.construct_version_query(input_json) logger.info(q) assert "access_points" in q assert "0.4.59" in q assert "pypi" in q
def create_pv_nodes(self): """Create Package and Version nodes, if needed.""" nodes = [ ] # return (e, p, v) tuples of created/existing nodes; for easier testing all_epvs_created = True for pv_dict in self._cve_dict.get('affected'): epv_dict = pv_dict.copy() epv_dict['ecosystem'] = self._cve_dict.get('ecosystem') query = GraphPopulator.construct_graph_nodes(epv_dict) success, json_response = BayesianGraph.execute(query) e = epv_dict.get('ecosystem') p = epv_dict.get('name') v = epv_dict.get('version') if not success: logger.error( 'CVEIngestionError - Error creating nodes for {e}/{p}/{v}: {r}' .format(e=e, p=p, v=v, r=str(json_response))) all_epvs_created = False else: nodes.append((e, p, v)) return nodes, all_epvs_created
def create_graph_nodes(list_epv): """Create blank graph nodes given an EPV.""" count_blank_epvs_created = 0 success_epvs = [] failure_epvs = [] for item in list_epv: str_gremlin, bindings = GraphPopulator.construct_graph_nodes(item) epv = item.get('ecosystem') + ":" + item.get('name') + ":" + item.get( 'version') if str_gremlin: payload = {'gremlin': str_gremlin, "bindings": bindings} print(json.dumps(payload)) try: result = requests.post(config.GREMLIN_SERVER_URL_REST, data=json.dumps(payload), timeout=30) resp = result.json() print(json.dumps(resp)) if resp['status']['code'] == 200: count_blank_epvs_created += 1 success_epvs.append(epv) except Exception as e: # pragma: no cover logger.error(e) failure_json = {epv: e} failure_epvs.append(failure_json) status = "Success" if count_blank_epvs_created == 0: status = "Failure" response = { "epv_nodes_created": count_blank_epvs_created, "success_list": success_epvs, "failure_list": failure_epvs, "status": status } return response
def _import_keys_from_s3_http(data_source, epv_list): # TODO: reduce cyclomatic complexity logger.debug("Begin import...") report = {'status': 'Success', 'message': 'The import finished successfully!'} count_imported_EPVs = 0 last_imported_EPV = None epv = [] for epv_key in epv_list: for key, contents in epv_key.items(): if len(contents.get('pkg_list_keys')) == 0 and len(contents.get('ver_list_keys')) == 0: report['message'] = 'Nothing to be imported! No data found on S3 to be imported!' continue pkg_ecosystem = contents.get('ecosystem') pkg_name = contents.get('package') pkg_version = contents.get('version') or '' pkg_source = contents.get('source_repo', pkg_ecosystem) obj = { 'ecosystem': pkg_ecosystem, 'package': pkg_name, 'version': pkg_version, 'source_repo': pkg_source} latest_version = get_latest_versions_for_ep(pkg_ecosystem, pkg_name) latest_epv_list = [{ 'ecosystem': pkg_ecosystem, 'name': pkg_name, 'version': latest_version }] create_graph_nodes(latest_epv_list) try: # Check other Version level information and add it to common object if len(contents.get('ver_list_keys')) > 0: first_key = contents['ver_key_prefix'] + '.json' first_obj = _first_key_info(data_source, first_key, config.AWS_EPV_BUCKET) first_obj['latest_version'] = latest_version obj.update(first_obj) ver_obj = _other_key_info(data_source, contents.get('ver_list_keys'), config.AWS_EPV_BUCKET) if 'analyses' in obj: obj.get('analyses', {}).update(ver_obj['analyses']) else: obj.update(ver_obj) # Check Package related information and add it to package object if len(contents.get('pkg_list_keys')) > 0: pkg_obj = _other_key_info(data_source, contents.get('pkg_list_keys'), config.AWS_PKG_BUCKET) if 'analyses' in obj: obj.get('analyses', {}).update(pkg_obj['analyses']) else: obj.update(pkg_obj) # Create Gremlin Query str_gremlin = GraphPopulator.create_query_string(obj) if str_gremlin: # Fire Gremlin HTTP query now epv_full = pkg_ecosystem + ":" + pkg_name + ":" + pkg_version logger.info("Ingestion initialized for EPV - %s" % epv_full) epv.append(epv_full) payload = {'gremlin': str_gremlin} response = requests.post(config.GREMLIN_SERVER_URL_REST, data=json.dumps(payload), timeout=30) resp = response.json() if resp['status']['code'] == 200: count_imported_EPVs += 1 last_imported_EPV = (obj.get('ecosystem') + ":" + obj.get('package') + ":" + obj.get('version')) # update first key with graph synced tag logger.info("Mark as synced in RDS %s" % last_imported_EPV) if not config.AWS_S3_IS_LOCAL: # pragma: no cover PostgresHandler().mark_epv_synced( obj.get('ecosystem'), obj.get('package'), obj.get('version') ) except Exception as e: # pragma: no cover logger.error(e) msg = _get_exception_msg("The import failed", e) report['status'] = 'Failure' report['message'] = msg report['epv'] = epv_key report['epv'] = epv_list report['count_imported_EPVs'] = count_imported_EPVs if count_imported_EPVs == 0 and report['status'] == 'Success': report['message'] = 'Nothing to be synced to Graph!' report['last_imported_EPV'] = last_imported_EPV return report
def create_pv_nodes(self): """Create Package and Version nodes, if needed.""" nodes = [] # return (e, p, v) tuples of created/existing nodes; for easier testing affected_pkgs = {} all_epvs_created = True p = self._snyk_pkg_data.get('package') e = self._snyk_pkg_data.get('ecosystem') latest_version = self._snyk_pkg_data.get('latest_version') latest_non_cve_version = '' epv_dict = { "ecosystem": e, "name": p, "latest_version": latest_version } if latest_version not in self._snyk_pkg_data.get('affected'): logger.info("Latest version is not affected {}".format(p)) latest_non_cve_version = latest_version else: logger.info("Latest version is affected {p} {v}".format(p=p, v=latest_version)) if e == 'golang': itr_list = self._snyk_pkg_data.get('all_ver') epv_dict['gh_link'] = self._snyk_pkg_data.get('gh_link') epv_dict['license'] = self._snyk_pkg_data.get('license') else: itr_list = self._snyk_pkg_data.get('affected') for ver in itr_list: epv_dict['version'] = ver query = GraphPopulator.construct_graph_nodes(epv_dict) success, json_response = BayesianGraph.execute(query) # Fetch the value of the latest_version from the query create if not latest_version and "latest_version" in query: data = query.split("\'latest_version\'")[1].split(");")[0] latest_version = data.replace(",", "").strip().replace("'", "") if not success: logger.error('CVEIngestionError - Error creating nodes for {e}/{p}/{v}: {r}'.format( e=e, p=p, v=ver, r=str(json_response)) ) all_epvs_created = False else: nodes.append((e, p, ver)) # To create the latest version node if not present if latest_version and latest_version != "-1" and e != "golang": epv_dict['version'] = latest_version logger.info("Creating latest version node {e} {p} {v}".format(e=epv_dict['ecosystem'], p=epv_dict['name'], v=epv_dict['version'])) query = GraphPopulator.construct_graph_nodes(epv_dict) BayesianGraph.execute(query) res = "" if latest_non_cve_version: res = update_non_cve_on_pkg(e, p, latest_non_cve_version) if p not in affected_pkgs and res != "Success": affected_pkg = { "ecosystem": e, "latest_version": latest_version } affected_pkgs[p] = affected_pkg return nodes, all_epvs_created, affected_pkgs
def handle_properties(ecosystem, package, version): """ Handle (update/delete) properties associated with given EPV. Update replaces properties with the same name. Expects JSON payload in following format: { "properties": [ { "name": "cve_ids", "value": "CVE-3005-0001:10" } ] } "value" can be omitted in DELETE requests. :param ecosystem: str, ecosystem :param package: str, package name :param version: str, package version :return: 200 on success, 400 on failure """ # TODO: reduce cyclomatic complexity input_json = request.get_json() properties = input_json.get('properties') error = flask.jsonify({'error': 'invalid input'}) if not properties: return error, 400 input_json = { k: GraphPopulator.sanitize_text_for_query(str(v)) for k, v in input_json.items() } if request.method == 'PUT': if [ x for x in properties if not x.get('name') or x.get('value') is None ]: return error, 400 log_msg = '[{m}] Updating properties for {e}/{p}/{v} with payload {b}' current_app.logger.info( log_msg.format(m=request.method, e=ecosystem, p=package, v=version, b=input_json)) query_statement = "g.V()" \ ".has('pecosystem','{ecosystem}')" \ ".has('pname','{pkg_name}')" \ ".has('version','{version}')".format(ecosystem=ecosystem, pkg_name=package, version=version) statement = '' if request.method in ('DELETE', 'PUT'): # build "delete" part of the statement drop_str = "" for prop in properties: drop_str += query_statement drop_str += ".properties('{property}').drop().iterate();".format( property=prop['name']) statement += drop_str if request.method == 'PUT': # build "add" part of the statement add_str = "" for prop in properties: add_str += ".property('{property}','{value}')".format( property=prop['name'], value=prop['value']) statement += query_statement + add_str + ';' current_app.logger.info('Gremlin statement: {s}'.format(s=statement)) success, response_json = BayesianGraph.execute(statement) if not success: current_app.logger.error( "Failed to update properties for {e}/{p}/{v}".format(e=ecosystem, p=package, v=version)) return flask.jsonify(response_json), 400 return flask.jsonify(response_json), 200
def test_construct_package_query(): """Test the GraphPopulator.construct_package_query() class method.""" input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": { "details": [{ "description": "Some description here" }] }, "github_details": {}, 'libraries_io': { 'schema': { 'version': '2-0-0' }, 'details': { 'releases': { 'count': 2, 'recent': [{ "published_at": "2016-09-09" }], "published_at": "2016-09-09" } } } } } str_package, prp_package = GraphPopulator.construct_package_query( input_json) logger.info(str_package) logger.info(prp_package) assert 'access_points' in str_package assert "pypi" in str_package input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": { "details": [{ "description": "Some description here" }] }, "github_details": {}, 'libraries_io': {} } } str_package, prp_package = GraphPopulator.construct_package_query( input_json) logger.info(str_package) logger.info(prp_package) assert 'access_points' in str_package assert "pypi" in str_package input_json = { "version": "0.4.59", "package": "access_points", "ecosystem": "pypi", "analyses": { "metadata": { "details": [{ "description": "Some description here" }] }, "github_details": {}, 'libraries_io': { 'schema': { 'version': '1-0-0' }, 'details': { 'releases': { 'count': 2, 'recent': [{ "published_at": "2016-09-09" }], 'latest': { 'recent': { "0.4.59": "2016-09-09" } }, "published_at": "2016-09-09" } } } } } str_package, prp_package = GraphPopulator.construct_package_query( input_json) logger.info(str_package) logger.info(prp_package) assert 'access_points' in str_package assert "pypi" in str_package