def get_repos(): """Read all the repo data.""" pkg_list = [] license_lst = [] eco_lst = [] for pkg in PACKAGE_DATA: pkg_list.append(PACKAGE_DATA[pkg]['name']) eco_lst.append(PACKAGE_DATA[pkg]['ecosystem']) query_str = "g.V().has('pecosystem',within(eco_lst))." \ "has('pname',within(pkg_list))" \ ".in('has_dependency').valueMap()" payload = { 'gremlin': query_str, 'bindings': { 'pkg_list': pkg_list, 'eco_lst': eco_lst } } gremlin_response = execute_gremlin_dsl(payload) result_data = get_response_data(gremlin_response, [{0: 0}]) repo_list = [] for data in result_data: repo_list.append(get_value(data, 'repo_url')) query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \ "out('has_dependency').as('b').select('a','b').by(valueMap())" payload = { 'gremlin': query_str, 'bindings': { 'repo_list': repo_list } } gremlin_response = execute_gremlin_dsl(payload) result_data = get_response_data(gremlin_response, [{0: 0}]) for result in result_data: repo = get_value(result['a'], 'repo_url') del license_lst[:] if 'licenses' in result['b']: licenses = result['b']['licenses'] for lic in licenses: license_lst.append(lic) eco = get_value(result['b'], 'pecosystem') name = get_value(result['b'], 'pname') version = get_value(result['b'], 'version') key = eco + ":" + name + ":" + version VERSION_DATA[key] = {} VERSION_DATA[key]['version'] = version VERSION_DATA[key]['package'] = eco + ":" + name VERSION_DATA[key]['license'] = license_lst if repo not in REPO_DATA: REPO_DATA[repo] = {} REPO_DATA[repo]['ecosystem'] = eco if 'dependencies' not in REPO_DATA[repo]: REPO_DATA[repo]['dependencies'] = [] key = eco + ":" + name + ":" + version if key not in REPO_DATA[repo]['dependencies']: REPO_DATA[repo]['dependencies'].append(key)
def test_execute_gremlin_dsl(mocker): """Test the function execute_gremlin_dsl.""" mocker.return_value = "" query_str = "g.V().has('ecosystem', eco).has('name',pkg).valueMap()" payload = { 'gremlin': query_str, 'bindings': { 'eco': 'maven', 'pkg': 'io.vertx:vertx-web' } } out = execute_gremlin_dsl(payload) assert out is None mocker.return_value = MockedSession("true") out = execute_gremlin_dsl(payload) assert out['requestId'] == "f98d1366-738e-4c14-a3ff-594f359e131c" out = get_response_data(out, [{0: 0}]) assert "a" in out mocker.return_value = MockedSession("false") query_str = "g.V().has('ecosystem', eco).has('name',pkg).valueMap()" payload = { 'gremlin': query_str, 'bindings': { 'eco': 'maven', 'pkg': 'io.vertx:vertx-web' } } out = execute_gremlin_dsl(payload) assert out is None
def get_etcd2_failover_info(): query = """ query { cluster { failover_params { mode state_provider etcd2_params { prefix lock_delay endpoints username password } failover_timeout fencing_enabled fencing_timeout fencing_pause } } } """ response = requests.post(get_admin_url(8081), json={'query': query}) return get_response_data(response)["cluster"]["failover_params"]
def get_version_information(input_list, ecosystem): """Fetch the version information for each of the packages. Also remove EPVs with CVEs and ones not present in Graph """ str_query = "data=[]; " for package in input_list: str_query += "pkg = g.V().has('ecosystem', '{eco}').has('name', '{pkg}'); " \ "lnv = []; pkg.clone().values('latest_non_cve_version', " \ "'latest_version').fill(lnv); pkg.clone().as('package').V()." \ "has('pecosystem', '{eco}').has('pname', '{pkg}')." \ "has('version', within(lnv)).as('version')." \ "select('package', 'version').by(valueMap()).fill(data);".format( eco=ecosystem, pkg=package) str_query += "data" payload = {'gremlin': str_query} # Query Gremlin with packages list to get their version information gremlin_response = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST, payload=payload) if gremlin_response is None: return [] response = get_response_data(gremlin_response, [{0: 0}]) return response
def read_packages(): """Read all the packages last updated.""" print("read_packages() started") prev_date = (datetime.utcnow() - timedelta(1)).strftime('%Y%m%d') query_str = "g.V().has('latest_version_last_updated',prev_date).valueMap()" # prev_date = '20180824' # for testing purpose, change date here payload = {'gremlin': query_str, 'bindings': {'prev_date': prev_date}} gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print( "Exception occured while trying to fetch packages : read_package") sys.exit() for result in result_data: eco = get_value(result, 'ecosystem') name = get_value(result, 'name') if not eco + ":" + name in PACKAGE_DATA: PACKAGE_DATA[eco + ":" + name] = {} PACKAGE_DATA[eco + ":" + name] = { 'name': name, 'ecosystem': eco, 'versions': [] } print("read_packages() ended")
def get_version_information(input_list, ecosystem): """Fetch the version information for each of the packages. Also remove EPVs with CVEs and ones not present in Graph """ input_packages = [package for package in input_list] str_query = "g.V().has('ecosystem',ecosystem).has('name',within(input_packages))" \ ".as('package').out('has_version')" \ ".not(outE('has_cve')).as('version').select('package','version')." \ "by(valueMap()).dedup()" payload = { 'gremlin': str_query, 'bindings': { 'ecosystem': ecosystem, 'input_packages': input_packages } } # Query Gremlin with packages list to get their version information gremlin_response = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST, payload=payload) if gremlin_response is None: return [] response = get_response_data(gremlin_response, [{0: 0}]) return response
def read_packages(): """Read all the packages last updated.""" prev_date = (datetime.utcnow() - timedelta(1)).strftime('%Y%m%d') query_str = "g.V().has('latest_version_last_updated',prev_date).valueMap()" # prev_date = '20180805' for testing purpose, change date here payload = { 'gremlin': query_str, 'bindings': { 'prev_date': prev_date } } gremlin_response = execute_gremlin_dsl(payload) result_data = get_response_data(gremlin_response, [{0: 0}]) for result in result_data: tmp_json = {} tmp_json['latest'] = get_value(result, 'latest_version') tmp_json['libio'] = get_value(result, 'libio_latest_version') eco = get_value(result, 'ecosystem') name = get_value(result, 'name') if not eco + ":" + name in PACKAGE_DATA: PACKAGE_DATA[eco + ":" + name] = {} tmp_json['name'] = name tmp_json['ecosystem'] = eco PACKAGE_DATA[eco + ":" + name] = tmp_json
def remove_cve_versions(): """Remove CVE versions.""" print("remove_cve_versions() started") pkg_list = [] ver_list = [] eco_lst = [] license_lst = [] for pkg in PACKAGE_DATA: if not PACKAGE_DATA[pkg]['name'] in pkg_list: pkg_list.append(PACKAGE_DATA[pkg]['name']) if not PACKAGE_DATA[pkg]['latest'] in ver_list: ver_list.append(PACKAGE_DATA[pkg]['latest']) if not PACKAGE_DATA[pkg]['libio'] in ver_list: ver_list.append(PACKAGE_DATA[pkg]['libio']) if not PACKAGE_DATA[pkg]['ecosystem'] in eco_lst: eco_lst.append(PACKAGE_DATA[pkg]['ecosystem']) query_str = "g.V().has('pecosystem',within(eco_lst))." \ "has('pname',within(pkg_list))" \ ".has('version',within(ver_list)).valueMap()" payload = { 'gremlin': query_str, 'bindings': { 'pkg_list': pkg_list, 'ver_list': ver_list, 'eco_lst': eco_lst } } gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print("Exception occured while trying to fetch versions : remove_cve_versions") sys.exit() for result in result_data: name = get_value(result, 'pname') eco = get_value(result, 'pecosystem') ver = get_value(result, 'version') if 'cve_ids' in result: if PACKAGE_DATA[eco + ":" + name]['latest'] == ver: del PACKAGE_DATA[eco + ":" + name]['latest'] elif PACKAGE_DATA[eco + ":" + name]['libio'] == ver: del PACKAGE_DATA[eco + ":" + name]['libio'] if 'libio' not in PACKAGE_DATA[eco + ":" + name] \ and 'latest' not in PACKAGE_DATA[eco + ":" + name]: del PACKAGE_DATA[eco + ":" + name] else: del license_lst[:] if 'licenses' in result: for lic in result['licenses']: license_lst.append(lic) key = eco + ":" + name + ":" + ver NEW_VERSION_DATA[key] = {} NEW_VERSION_DATA[key]['version'] = ver NEW_VERSION_DATA[key]['package'] = eco + ":" + name NEW_VERSION_DATA[key]['license'] = license_lst print("remove_cve_versions() ended")
def get_eventual_failover_info(): query = """ query { cluster { failover_params { mode failover_timeout fencing_enabled fencing_timeout fencing_pause } } } """ response = requests.post(get_admin_url(8081), json={'query': query}) return get_response_data(response)["cluster"]["failover_params"]
def get_stateboard_failover_info(): query = """ query { cluster { failover_params { mode state_provider tarantool_params { uri password } failover_timeout fencing_enabled fencing_timeout fencing_pause } } } """ response = requests.post(get_admin_url(8081), json={'query': query}) return get_response_data(response)["cluster"]["failover_params"]
def get_version_data(pkg_data, new_ver_data, version_data, tr_flag="false"): """Get all the version info for the packages.""" print("get_version_data() started") pkg_list = [] eco_list = [] license_list = [] for repo in REPO_DATA: if tr_flag is "true" and "tr_dependencies" in REPO_DATA[repo]: deps = REPO_DATA[repo]['tr_dependencies'] elif tr_flag is "false": deps = REPO_DATA[repo]['dependencies'] else: continue for dep in deps: dep_data = version_data[dep] pkg_list.append(dep_data['name']) eco_list.append(REPO_DATA[repo]['ecosystem']) query_str = "g.V().has('pecosystem',within(eco_list))." \ "has('pname',within(pkg_list))" \ ".hasNot('cve_ids').valueMap().dedup()" payload = { 'gremlin': query_str, 'bindings': { 'pkg_list': list(set(pkg_list)), 'eco_list': list(set(eco_list)) } } gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print( "Exception occured while trying to fetch versions : remove_cve_versions" ) sys.exit() for result in result_data: name = get_value(result, 'pname') eco = get_value(result, 'pecosystem') ver = get_value(result, 'version') pkg_key = eco + ":" + name del license_list[:] if 'licenses' in result: for lic in result['licenses']: license_list.append(lic) key = eco + ":" + name + ":" + ver new_ver_data[key] = { 'version': ver, 'package': eco + ":" + name, 'license': license_list } if pkg_key not in pkg_data: pkg_data[pkg_key] = { 'name': name, 'ecosystem': eco, 'versions': [] } pkg_data[pkg_key]['versions'].append(key) print("get_version_data() ended")
def get_repos(): """Read all the repo data.""" print("get_repos() started") pkg_list = [] license_list = [] eco_list = [] for pkg in PACKAGE_DATA: pkg_list.append(PACKAGE_DATA[pkg]['name']) eco_list.append(PACKAGE_DATA[pkg]['ecosystem']) query_str = "g.V().has('pecosystem',within(eco_list))." \ "has('pname',within(pkg_list))" \ ".in('has_dependency').valueMap()" payload = { 'gremlin': query_str, 'bindings': { 'pkg_list': pkg_list, 'eco_list': eco_list } } gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print("Exception occured while trying to fetch repo : get_repos") sys.exit() repo_list = [] for data in result_data: repo_list.append(get_value(data, 'repo_url')) query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \ "out('has_dependency').as('b').select('a','b').by(valueMap())" payload = {'gremlin': query_str, 'bindings': {'repo_list': repo_list}} gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print("Exception occured while trying to fetch versions : get_repos") sys.exit() for result in result_data: repo = get_value(result['a'], 'repo_url') del license_list[:] if 'licenses' in result['b']: licenses = result['b']['licenses'] for lic in licenses: license_list.append(lic) eco = get_value(result['b'], 'pecosystem') name = get_value(result['b'], 'pname') version = get_value(result['b'], 'version') key = eco + ":" + name + ":" + version VERSION_DATA[key] = { 'version': version, 'name': name, 'package': eco + ":" + name, 'license': license_list } if repo not in REPO_DATA: REPO_DATA[repo] = {} REPO_DATA[repo]['ecosystem'] = eco if 'dependencies' not in REPO_DATA[repo]: REPO_DATA[repo]['dependencies'] = [] key = eco + ":" + name + ":" + version if key not in REPO_DATA[repo]['dependencies']: REPO_DATA[repo]['dependencies'].append(key) query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \ "out('has_transitive_dependency').as('b').select('a','b').by(valueMap())" payload = {'gremlin': query_str, 'bindings': {'repo_list': repo_list}} gremlin_response = execute_gremlin_dsl(payload) if gremlin_response is not None: result_data = get_response_data(gremlin_response, [{0: 0}]) else: print("Exception occured while trying to fetch versions : get_repos") sys.exit() for result in result_data: repo = get_value(result['a'], 'repo_url') eco = get_value(result['b'], 'pecosystem') name = get_value(result['b'], 'pname') version = get_value(result['b'], 'version') key = eco + ":" + name + ":" + version TRANSITIVE_VERSION_DATA[key] = { 'version': version, 'name': name, 'package': eco + ":" + name } pkg_key = eco + ":" + name TRANSITIVE_PACKAGE_DATA[pkg_key] = { 'ecosystem': eco, 'name': name, 'versions': [] } if repo not in REPO_DATA: REPO_DATA[repo] = {} REPO_DATA[repo]['ecosystem'] = eco if 'tr_dependencies' not in REPO_DATA[repo]: REPO_DATA[repo]['tr_dependencies'] = [] key = eco + ":" + name + ":" + version if key not in REPO_DATA[repo]['tr_dependencies']: REPO_DATA[repo]['tr_dependencies'].append(key) print("get_repos() ended")