def get_repos():
"""Read all the repo data."""
pkg_list = []
license_lst = []
eco_lst = []
for pkg in PACKAGE_DATA:
pkg_list.append(PACKAGE_DATA[pkg]['name'])
eco_lst.append(PACKAGE_DATA[pkg]['ecosystem'])
query_str = "g.V().has('pecosystem',within(eco_lst))." \
"has('pname',within(pkg_list))" \
".in('has_dependency').valueMap()"
payload = {
'gremlin': query_str,
'bindings': {
'pkg_list': pkg_list,
'eco_lst': eco_lst
}
}
gremlin_response = execute_gremlin_dsl(payload)
result_data = get_response_data(gremlin_response, [{0: 0}])
repo_list = []
for data in result_data:
repo_list.append(get_value(data, 'repo_url'))
query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \
"out('has_dependency').as('b').select('a','b').by(valueMap())"
payload = {
'gremlin': query_str,
'bindings': {
'repo_list': repo_list
}
}
gremlin_response = execute_gremlin_dsl(payload)
result_data = get_response_data(gremlin_response, [{0: 0}])
for result in result_data:
repo = get_value(result['a'], 'repo_url')
del license_lst[:]
if 'licenses' in result['b']:
licenses = result['b']['licenses']
for lic in licenses:
license_lst.append(lic)
eco = get_value(result['b'], 'pecosystem')
name = get_value(result['b'], 'pname')
version = get_value(result['b'], 'version')
key = eco + ":" + name + ":" + version
VERSION_DATA[key] = {}
VERSION_DATA[key]['version'] = version
VERSION_DATA[key]['package'] = eco + ":" + name
VERSION_DATA[key]['license'] = license_lst
if repo not in REPO_DATA:
REPO_DATA[repo] = {}
REPO_DATA[repo]['ecosystem'] = eco
if 'dependencies' not in REPO_DATA[repo]:
REPO_DATA[repo]['dependencies'] = []
key = eco + ":" + name + ":" + version
if key not in REPO_DATA[repo]['dependencies']:
REPO_DATA[repo]['dependencies'].append(key)
def test_execute_gremlin_dsl(mocker):
"""Test the function execute_gremlin_dsl."""
mocker.return_value = ""
query_str = "g.V().has('ecosystem', eco).has('name',pkg).valueMap()"
payload = {
'gremlin': query_str,
'bindings': {
'eco': 'maven',
'pkg': 'io.vertx:vertx-web'
}
}
out = execute_gremlin_dsl(payload)
assert out is None
mocker.return_value = MockedSession("true")
out = execute_gremlin_dsl(payload)
assert out['requestId'] == "f98d1366-738e-4c14-a3ff-594f359e131c"
out = get_response_data(out, [{0: 0}])
assert "a" in out
mocker.return_value = MockedSession("false")
query_str = "g.V().has('ecosystem', eco).has('name',pkg).valueMap()"
payload = {
'gremlin': query_str,
'bindings': {
'eco': 'maven',
'pkg': 'io.vertx:vertx-web'
}
}
out = execute_gremlin_dsl(payload)
assert out is None
def get_etcd2_failover_info():
query = """
query {
cluster {
failover_params {
mode
state_provider
etcd2_params {
prefix
lock_delay
endpoints
username
password
}
failover_timeout
fencing_enabled
fencing_timeout
fencing_pause
}
}
}
"""
response = requests.post(get_admin_url(8081), json={'query': query})
return get_response_data(response)["cluster"]["failover_params"]
def get_version_information(input_list, ecosystem):
"""Fetch the version information for each of the packages.
Also remove EPVs with CVEs and ones not present in Graph
"""
str_query = "data=[]; "
for package in input_list:
str_query += "pkg = g.V().has('ecosystem', '{eco}').has('name', '{pkg}'); " \
"lnv = []; pkg.clone().values('latest_non_cve_version', " \
"'latest_version').fill(lnv); pkg.clone().as('package').V()." \
"has('pecosystem', '{eco}').has('pname', '{pkg}')." \
"has('version', within(lnv)).as('version')." \
"select('package', 'version').by(valueMap()).fill(data);".format(
eco=ecosystem,
pkg=package)
str_query += "data"
payload = {'gremlin': str_query}
# Query Gremlin with packages list to get their version information
gremlin_response = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST,
payload=payload)
if gremlin_response is None:
return []
response = get_response_data(gremlin_response, [{0: 0}])
return response
def read_packages():
"""Read all the packages last updated."""
print("read_packages() started")
prev_date = (datetime.utcnow() - timedelta(1)).strftime('%Y%m%d')
query_str = "g.V().has('latest_version_last_updated',prev_date).valueMap()"
# prev_date = '20180824' # for testing purpose, change date here
payload = {'gremlin': query_str, 'bindings': {'prev_date': prev_date}}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print(
"Exception occured while trying to fetch packages : read_package")
sys.exit()
for result in result_data:
eco = get_value(result, 'ecosystem')
name = get_value(result, 'name')
if not eco + ":" + name in PACKAGE_DATA:
PACKAGE_DATA[eco + ":" + name] = {}
PACKAGE_DATA[eco + ":" + name] = {
'name': name,
'ecosystem': eco,
'versions': []
}
print("read_packages() ended")
def get_version_information(input_list, ecosystem):
"""Fetch the version information for each of the packages.
Also remove EPVs with CVEs and ones not present in Graph
"""
input_packages = [package for package in input_list]
str_query = "g.V().has('ecosystem',ecosystem).has('name',within(input_packages))" \
".as('package').out('has_version')" \
".not(outE('has_cve')).as('version').select('package','version')." \
"by(valueMap()).dedup()"
payload = {
'gremlin': str_query,
'bindings': {
'ecosystem': ecosystem,
'input_packages': input_packages
}
}
# Query Gremlin with packages list to get their version information
gremlin_response = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST,
payload=payload)
if gremlin_response is None:
return []
response = get_response_data(gremlin_response, [{0: 0}])
return response
def read_packages():
"""Read all the packages last updated."""
prev_date = (datetime.utcnow() - timedelta(1)).strftime('%Y%m%d')
query_str = "g.V().has('latest_version_last_updated',prev_date).valueMap()"
# prev_date = '20180805' for testing purpose, change date here
payload = {
'gremlin': query_str,
'bindings': {
'prev_date': prev_date
}
}
gremlin_response = execute_gremlin_dsl(payload)
result_data = get_response_data(gremlin_response, [{0: 0}])
for result in result_data:
tmp_json = {}
tmp_json['latest'] = get_value(result, 'latest_version')
tmp_json['libio'] = get_value(result, 'libio_latest_version')
eco = get_value(result, 'ecosystem')
name = get_value(result, 'name')
if not eco + ":" + name in PACKAGE_DATA:
PACKAGE_DATA[eco + ":" + name] = {}
tmp_json['name'] = name
tmp_json['ecosystem'] = eco
PACKAGE_DATA[eco + ":" + name] = tmp_json
def remove_cve_versions():
"""Remove CVE versions."""
print("remove_cve_versions() started")
pkg_list = []
ver_list = []
eco_lst = []
license_lst = []
for pkg in PACKAGE_DATA:
if not PACKAGE_DATA[pkg]['name'] in pkg_list:
pkg_list.append(PACKAGE_DATA[pkg]['name'])
if not PACKAGE_DATA[pkg]['latest'] in ver_list:
ver_list.append(PACKAGE_DATA[pkg]['latest'])
if not PACKAGE_DATA[pkg]['libio'] in ver_list:
ver_list.append(PACKAGE_DATA[pkg]['libio'])
if not PACKAGE_DATA[pkg]['ecosystem'] in eco_lst:
eco_lst.append(PACKAGE_DATA[pkg]['ecosystem'])
query_str = "g.V().has('pecosystem',within(eco_lst))." \
"has('pname',within(pkg_list))" \
".has('version',within(ver_list)).valueMap()"
payload = {
'gremlin': query_str,
'bindings': {
'pkg_list': pkg_list,
'ver_list': ver_list,
'eco_lst': eco_lst
}
}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print("Exception occured while trying to fetch versions : remove_cve_versions")
sys.exit()
for result in result_data:
name = get_value(result, 'pname')
eco = get_value(result, 'pecosystem')
ver = get_value(result, 'version')
if 'cve_ids' in result:
if PACKAGE_DATA[eco + ":" + name]['latest'] == ver:
del PACKAGE_DATA[eco + ":" + name]['latest']
elif PACKAGE_DATA[eco + ":" + name]['libio'] == ver:
del PACKAGE_DATA[eco + ":" + name]['libio']
if 'libio' not in PACKAGE_DATA[eco + ":" + name] \
and 'latest' not in PACKAGE_DATA[eco + ":" + name]:
del PACKAGE_DATA[eco + ":" + name]
else:
del license_lst[:]
if 'licenses' in result:
for lic in result['licenses']:
license_lst.append(lic)
key = eco + ":" + name + ":" + ver
NEW_VERSION_DATA[key] = {}
NEW_VERSION_DATA[key]['version'] = ver
NEW_VERSION_DATA[key]['package'] = eco + ":" + name
NEW_VERSION_DATA[key]['license'] = license_lst
print("remove_cve_versions() ended")
def get_eventual_failover_info():
query = """
query {
cluster {
failover_params {
mode
failover_timeout
fencing_enabled
fencing_timeout
fencing_pause
}
}
}
"""
response = requests.post(get_admin_url(8081), json={'query': query})
return get_response_data(response)["cluster"]["failover_params"]
def get_stateboard_failover_info():
query = """
query {
cluster {
failover_params {
mode
state_provider
tarantool_params {
uri
password
}
failover_timeout
fencing_enabled
fencing_timeout
fencing_pause
}
}
}
"""
response = requests.post(get_admin_url(8081), json={'query': query})
return get_response_data(response)["cluster"]["failover_params"]
def get_version_data(pkg_data, new_ver_data, version_data, tr_flag="false"):
"""Get all the version info for the packages."""
print("get_version_data() started")
pkg_list = []
eco_list = []
license_list = []
for repo in REPO_DATA:
if tr_flag is "true" and "tr_dependencies" in REPO_DATA[repo]:
deps = REPO_DATA[repo]['tr_dependencies']
elif tr_flag is "false":
deps = REPO_DATA[repo]['dependencies']
else:
continue
for dep in deps:
dep_data = version_data[dep]
pkg_list.append(dep_data['name'])
eco_list.append(REPO_DATA[repo]['ecosystem'])
query_str = "g.V().has('pecosystem',within(eco_list))." \
"has('pname',within(pkg_list))" \
".hasNot('cve_ids').valueMap().dedup()"
payload = {
'gremlin': query_str,
'bindings': {
'pkg_list': list(set(pkg_list)),
'eco_list': list(set(eco_list))
}
}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print(
"Exception occured while trying to fetch versions : remove_cve_versions"
)
sys.exit()
for result in result_data:
name = get_value(result, 'pname')
eco = get_value(result, 'pecosystem')
ver = get_value(result, 'version')
pkg_key = eco + ":" + name
del license_list[:]
if 'licenses' in result:
for lic in result['licenses']:
license_list.append(lic)
key = eco + ":" + name + ":" + ver
new_ver_data[key] = {
'version': ver,
'package': eco + ":" + name,
'license': license_list
}
if pkg_key not in pkg_data:
pkg_data[pkg_key] = {
'name': name,
'ecosystem': eco,
'versions': []
}
pkg_data[pkg_key]['versions'].append(key)
print("get_version_data() ended")
def get_repos():
"""Read all the repo data."""
print("get_repos() started")
pkg_list = []
license_list = []
eco_list = []
for pkg in PACKAGE_DATA:
pkg_list.append(PACKAGE_DATA[pkg]['name'])
eco_list.append(PACKAGE_DATA[pkg]['ecosystem'])
query_str = "g.V().has('pecosystem',within(eco_list))." \
"has('pname',within(pkg_list))" \
".in('has_dependency').valueMap()"
payload = {
'gremlin': query_str,
'bindings': {
'pkg_list': pkg_list,
'eco_list': eco_list
}
}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print("Exception occured while trying to fetch repo : get_repos")
sys.exit()
repo_list = []
for data in result_data:
repo_list.append(get_value(data, 'repo_url'))
query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \
"out('has_dependency').as('b').select('a','b').by(valueMap())"
payload = {'gremlin': query_str, 'bindings': {'repo_list': repo_list}}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print("Exception occured while trying to fetch versions : get_repos")
sys.exit()
for result in result_data:
repo = get_value(result['a'], 'repo_url')
del license_list[:]
if 'licenses' in result['b']:
licenses = result['b']['licenses']
for lic in licenses:
license_list.append(lic)
eco = get_value(result['b'], 'pecosystem')
name = get_value(result['b'], 'pname')
version = get_value(result['b'], 'version')
key = eco + ":" + name + ":" + version
VERSION_DATA[key] = {
'version': version,
'name': name,
'package': eco + ":" + name,
'license': license_list
}
if repo not in REPO_DATA:
REPO_DATA[repo] = {}
REPO_DATA[repo]['ecosystem'] = eco
if 'dependencies' not in REPO_DATA[repo]:
REPO_DATA[repo]['dependencies'] = []
key = eco + ":" + name + ":" + version
if key not in REPO_DATA[repo]['dependencies']:
REPO_DATA[repo]['dependencies'].append(key)
query_str = "g.V().has('repo_url', within(repo_list)).as('a')." \
"out('has_transitive_dependency').as('b').select('a','b').by(valueMap())"
payload = {'gremlin': query_str, 'bindings': {'repo_list': repo_list}}
gremlin_response = execute_gremlin_dsl(payload)
if gremlin_response is not None:
result_data = get_response_data(gremlin_response, [{0: 0}])
else:
print("Exception occured while trying to fetch versions : get_repos")
sys.exit()
for result in result_data:
repo = get_value(result['a'], 'repo_url')
eco = get_value(result['b'], 'pecosystem')
name = get_value(result['b'], 'pname')
version = get_value(result['b'], 'version')
key = eco + ":" + name + ":" + version
TRANSITIVE_VERSION_DATA[key] = {
'version': version,
'name': name,
'package': eco + ":" + name
}
pkg_key = eco + ":" + name
TRANSITIVE_PACKAGE_DATA[pkg_key] = {
'ecosystem': eco,
'name': name,
'versions': []
}
if repo not in REPO_DATA:
REPO_DATA[repo] = {}
REPO_DATA[repo]['ecosystem'] = eco
if 'tr_dependencies' not in REPO_DATA[repo]:
REPO_DATA[repo]['tr_dependencies'] = []
key = eco + ":" + name + ":" + version
if key not in REPO_DATA[repo]['tr_dependencies']:
REPO_DATA[repo]['tr_dependencies'].append(key)
print("get_repos() ended")
