def find_latest_version():
"""Find the latest version."""
tmp_lst = []
for repo in REPO_DATA:
del tmp_lst[:]
FINAL_DATA[repo] = {}
FINAL_DATA[repo]['notify'] = "false"
latest_ver = ''
libio_ver = ''
deps = REPO_DATA[repo]['dependencies']
for dep in deps:
tmp_json = {}
pkg = VERSION_DATA[dep]['package']
if pkg in PACKAGE_DATA and \
(dep in NEW_VERSION_DATA or dep in VERSION_DATA):
if 'latest' in PACKAGE_DATA[pkg]:
latest_ver = PACKAGE_DATA[pkg]['latest']
if 'libio' in PACKAGE_DATA[pkg]:
libio_ver = PACKAGE_DATA[pkg]['libio']
cur_ver = VERSION_DATA[dep]['version']
pkg_name = PACKAGE_DATA[pkg]['name']
latest_version = \
select_latest_version(cur_ver,
libio_ver,
latest_ver,
pkg_name)
if latest_version != cur_ver:
FINAL_DATA[repo]['notify'] = "true"
tmp_json['ecosystem'] = REPO_DATA[repo]['ecosystem']
tmp_json['name'] = pkg_name
tmp_json['version'] = cur_ver
tmp_json['latest_version'] = latest_version
tmp_lst.append(tmp_json)
FINAL_DATA[repo]['version_updates'] = tmp_lst[:]
def test_select_latest_version():
"""Test the function select_latest_version."""
lat_ver = select_latest_version("-1", ["3.4.5", "3.4.1"], "pkg")
assert lat_ver is "3.4.5"
lat_ver = select_latest_version("-1", ["3.4.1", "3.4.5"], "pkg")
assert lat_ver is "3.4.5"
lat_ver = select_latest_version("3.4.5", ["3.4.1", "3.4.0"], "pkg")
assert lat_ver is "3.4.5"
lat_ver = select_latest_version("-1", ["-1", "-1"], "pkg")
assert lat_ver is ''
lat_ver = select_latest_version(["abc"], [{"a": "b"}, ["b"]], "pkg")
assert lat_ver is ''
def get_recommended_version(ecosystem, name, version):
"""Fetch the recommended version in case of CVEs."""
query = "g.V().has('ecosystem', '{eco}').has('name', '{pkg}')" \
".out('has_version').not(out('has_cve')).values('version');"\
.format(eco=ecosystem, pkg=name)
payload = {'gremlin': query}
result = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST, payload=payload)
if result:
versions = result['result']['data']
if len(versions) == 0:
return None
else:
return None
rec_version = version
for ver in versions:
rec_version = select_latest_version(ver, rec_version)
if rec_version == version:
return None
return rec_version
def filter_versions(epv_list,
input_stack,
external_request_id=None,
rec_type=None):
"""Filter the EPVs according to following rules.
First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date
"""
# TODO: reduce cyclomatic complexity
logger.info("Filtering {} for external_request_id {}".format(
rec_type, external_request_id))
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
# TODO: refactoring
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
# needed for maven version like 1.5.2.RELEASE to be converted to
# 1.5.2-RELEASE for semantic version to work'
semversion_tuple = version_info_tuple(
convert_version_to_proper_semantic(version, name))
input_stack_tuple = version_info_tuple(
convert_version_to_proper_semantic(input_stack.get(name, ''),
name))
if name and version:
# Select Latest Version and add to filter_list if
# latest version is > current version
latest_version = select_latest_version(
version,
epv.get('pkg').get('libio_latest_version', [''])[0],
epv.get('pkg').get('latest_version', [''])[0], name)
if latest_version and latest_version == version:
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['latest_version'] = latest_version
new_dict[name]['latest_version'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering latest version!"
)
pass
# Check for Dependency Count Attribute. Add Max deps count version
# if version > current version
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get('deps_count', {}).get(
'deps_count', 0):
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['deps_count'] = {
"version": version,
"deps_count": deps_count
}
new_dict[name]['deps_count'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering dependency count!"
)
pass
# Check for github release date. Add version with most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > \
pkg_dict[name].get('gh_release_date', {}).get('gh_release_date',
0):
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['gh_release_date'] = {
"version": version,
"gh_release_date": gh_release_date
}
new_dict[name]['gh_release_date'] = epv.get(
'ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering github release date!"
)
pass
logger.info(
"Data Dict new_dict for external_request_id {} is {}".format(
external_request_id, new_dict))
logger.info(
"Data List filtered_comp_list for external_request_id {} is {}".
format(external_request_id, filtered_comp_list))
# TODO: refactoring
new_list = []
for package, contents in new_dict.items():
if 'latest_version' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['latest_version']
})
elif 'deps_count' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['deps_count']
})
elif 'gh_release_date' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['gh_release_date']
})
return new_list, filtered_comp_list
def filter_versions(epv_list, input_stack, external_request_id=None, rec_type=None):
"""Filter the EPVs according to following rules.
First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date
"""
logger.info("Filtering {} for external_request_id {}".format(rec_type, external_request_id))
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
libio_latest_version = epv.get('pkg').get('libio_latest_version', [''])[0]
latest_version = epv.get('pkg').get('latest_version', [''])[0]
# Convert version to a proper semantic case
semversion_tuple = version_info_tuple(
convert_version_to_proper_semantic(version, name))
input_stack_tuple = version_info_tuple(
convert_version_to_proper_semantic(input_stack.get(name, ''), name))
if name and version:
# Select highest version based on input or graph as latest version
latest_version = select_latest_version(
version, libio_latest_version, latest_version, name
)
if latest_version and latest_version == version:
pkg_dict, new_dict, filtered_comp_list = GraphDB.add_version_to_filtered_list(
epv=epv, key='latest_version', val=latest_version, pkg_dict=pkg_dict,
new_dict=new_dict, filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple)
# Select Version based on highest dependents count (usage)
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get['deps_count'].get('deps_count', 0):
pkg_dict, new_dict, filtered_comp_list = \
GraphDB.add_version_to_filtered_list(
epv=epv, key='deps_count', val=deps_count, pkg_dict=pkg_dict,
new_dict=new_dict, filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple,
input_stack_tuple=input_stack_tuple)
# Select Version with the most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0.0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > pkg_dict[name]['gh_release_date'].\
get('gh_release_date', 0.0):
pkg_dict, new_dict, filtered_comp_list = \
GraphDB.add_version_to_filtered_list(
epv=epv, key='gh_release_date', val=gh_release_date,
pkg_dict=pkg_dict, new_dict=new_dict,
filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple,
input_stack_tuple=input_stack_tuple)
logger.info("Data Dict new_dict for external_request_id {} is {}".format(
external_request_id, new_dict))
logger.info("Data List filtered_comp_list for external_request_id {} is {}".format(
external_request_id, filtered_comp_list))
new_list = GraphDB.prepare_final_filtered_list(new_dict)
return new_list, filtered_comp_list
def find_latest_version(pkg_data,
version_data,
new_version_data,
transitive_flag="false"):
"""Find the latest version."""
print("find_latest_version() started")
tr_list = []
dir_list = []
for repo in REPO_DATA:
del tr_list[:]
del dir_list[:]
if transitive_flag is "false":
deps = REPO_DATA[repo]['dependencies']
FINAL_DATA[repo] = {
'notify': 'false',
'transitive_updates': [],
'direct_updates': []
}
else:
if 'tr_dependencies' in REPO_DATA[repo]:
deps = REPO_DATA[repo]['tr_dependencies']
else:
deps = []
for dep in deps:
tmp_json = {}
ver_list = []
pkg = version_data[dep]['package']
if pkg in pkg_data and \
(dep in new_version_data or dep in version_data) and \
(transitive_flag is "false" or (
FINAL_DATA[repo]['notify'] is "true" and transitive_flag is "true")):
if 'versions' in pkg_data[pkg]:
versions = pkg_data[pkg]['versions']
for ver in versions:
ver_list.append(new_version_data[ver]['version'])
cur_ver = version_data[dep]['version']
pkg_name = pkg_data[pkg]['name']
latest_version = \
select_latest_version(cur_ver,
ver_list,
pkg_name)
eco = REPO_DATA[repo]['ecosystem']
latest_key = eco + ":" + pkg_name + ":" + latest_version
if latest_version != cur_ver and \
(latest_key in new_version_data or latest_key in version_data):
FINAL_DATA[repo]['notify'] = "true"
tmp_json = {
'ecosystem': eco,
'name': pkg_name,
'version': cur_ver,
'latest_version': latest_version
}
if transitive_flag is "true":
tr_list.append(tmp_json)
else:
dir_list.append(tmp_json)
if tr_list:
FINAL_DATA[repo]['transitive_updates'].extend(tr_list[:])
if dir_list:
FINAL_DATA[repo]['direct_updates'].extend(dir_list[:])
print("find_latest_version() ended")
def extract_component_details(component):
github_details = {
"dependent_projects":
component.get("package", {}).get("libio_dependents_projects", [-1])[0],
"dependent_repos":
component.get("package", {}).get("libio_dependents_repos", [-1])[0],
"total_releases":
component.get("package", {}).get("libio_total_releases", [-1])[0],
"latest_release_duration":
str(
datetime.datetime.fromtimestamp(
component.get("package", {}).get("libio_latest_release",
[1496302486.0])[0])),
"first_release_date":
"Apr 16, 2010",
"issues": {
"month": {
"opened":
component.get("package", {}).get("gh_issues_last_month_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_issues_last_month_closed",
[-1])[0]
},
"year": {
"opened":
component.get("package", {}).get("gh_issues_last_year_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_issues_last_year_closed",
[-1])[0]
}
},
"pull_requests": {
"month": {
"opened":
component.get("package", {}).get("gh_prs_last_month_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_prs_last_month_closed",
[-1])[0]
},
"year": {
"opened":
component.get("package", {}).get("gh_prs_last_year_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_prs_last_year_closed",
[-1])[0]
}
},
"stargazers_count":
component.get("package", {}).get("gh_stargazers", [-1])[0],
"forks_count":
component.get("package", {}).get("gh_forks", [-1])[0],
"open_issues_count":
component.get("package", {}).get("gh_open_issues_count", [-1])[0],
"contributors":
component.get("package", {}).get("gh_contributors_count", [-1])[0],
"size":
"N/A"
}
used_by = component.get("package", {}).get("libio_usedby", [])
used_by_list = []
for epvs in used_by:
slc = epvs.split(':')
used_by_dict = {'name': slc[0], 'stars': int(slc[1])}
used_by_list.append(used_by_dict)
github_details['used_by'] = used_by_list
code_metrics = {
"code_lines":
component.get("version", {}).get("cm_loc", [-1])[0],
"average_cyclomatic_complexity":
component.get("version", {}).get("cm_avg_cyclomatic_complexity",
[-1])[0],
"total_files":
component.get("version", {}).get("cm_num_files", [-1])[0]
}
cves = []
for cve in component.get("version", {}).get("cve_ids", []):
component_cve = {'CVE': cve.split(':')[0], 'CVSS': cve.split(':')[1]}
cves.append(component_cve)
licenses = component.get("version", {}).get("declared_licenses", [])
name = component.get("version", {}).get("pname", [""])[0]
version = component.get("version", {}).get("version", [""])[0]
ecosystem = component.get("version", {}).get("pecosystem", [""])[0]
latest_version = select_latest_version(
version,
component.get("package", {}).get("libio_latest_version", [""])[0],
component.get("package", {}).get("latest_version", [""])[0])
component_summary = {
"ecosystem": ecosystem,
"name": name,
"version": version,
"licenses": licenses,
"security": cves,
"osio_user_count": component.get("version",
{}).get("osio_usage_count", 0),
"latest_version": latest_version,
"github": github_details,
"code_metrics": code_metrics
}
return component_summary
def extract_component_details(component):
"""Extract details from given component."""
date = format_date(
component.get("package", {}).get("gh_refreshed_on", ["N/A"])[0])
github_details = {
"dependent_projects":
component.get("package", {}).get("libio_dependents_projects", [-1])[0],
"dependent_repos":
component.get("package", {}).get("libio_dependents_repos", [-1])[0],
"total_releases":
component.get("package", {}).get("libio_total_releases", [-1])[0],
"latest_release_duration":
str(
datetime.datetime.fromtimestamp(
component.get("package", {}).get("libio_latest_release",
[1496302486.0])[0])),
"first_release_date":
"Apr 16, 2010",
"issues": {
"month": {
"opened":
component.get("package", {}).get("gh_issues_last_month_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_issues_last_month_closed",
[-1])[0]
},
"year": {
"opened":
component.get("package", {}).get("gh_issues_last_year_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_issues_last_year_closed",
[-1])[0]
}
},
"pull_requests": {
"month": {
"opened":
component.get("package", {}).get("gh_prs_last_month_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_prs_last_month_closed",
[-1])[0]
},
"year": {
"opened":
component.get("package", {}).get("gh_prs_last_year_opened",
[-1])[0],
"closed":
component.get("package", {}).get("gh_prs_last_year_closed",
[-1])[0]
}
},
"stargazers_count":
component.get("package", {}).get("gh_stargazers", [-1])[0],
"forks_count":
component.get("package", {}).get("gh_forks", [-1])[0],
"refreshed_on":
date,
"open_issues_count":
component.get("package", {}).get("gh_open_issues_count", [-1])[0],
"contributors":
component.get("package", {}).get("gh_contributors_count", [-1])[0],
"size":
"N/A"
}
used_by = component.get("package", {}).get("libio_usedby", [])
used_by_list = []
for epvs in used_by:
slc = epvs.split(':')
used_by_dict = {'name': slc[0], 'stars': int(slc[1])}
used_by_list.append(used_by_dict)
github_details['used_by'] = used_by_list
code_metrics = {
"code_lines":
component.get("version", {}).get("cm_loc", [-1])[0],
"average_cyclomatic_complexity":
component.get("version", {}).get("cm_avg_cyclomatic_complexity",
[-1])[0],
"total_files":
component.get("version", {}).get("cm_num_files", [-1])[0]
}
cves = []
recommended_latest_version = None
name = component.get("version", {}).get("pname", [""])[0]
version = component.get("version", {}).get("version", [""])[0]
ecosystem = component.get("version", {}).get("pecosystem", [""])[0]
if len(component.get("cves", [])) > 0:
for cve in component.get("cves", []):
component_cve = {
'CVE': cve.get('cve_id')[0],
'CVSS': cve.get('cvss_v2', [''])[0]
}
cves.append(component_cve)
recommended_latest_version = component.get("package", {}).get(
"latest_non_cve_version", "")
if not recommended_latest_version:
recommended_latest_version = get_recommended_version(
ecosystem, name, version)
licenses = component.get("version", {}).get("declared_licenses", [])
latest_version = select_latest_version(
version,
component.get("package", {}).get("libio_latest_version", [""])[0],
component.get("package", {}).get("latest_version", [""])[0], name)
component_summary = {
"ecosystem": ecosystem,
"name": name,
"version": version,
"licenses": licenses,
"security": cves,
"osio_user_count": component.get("version",
{}).get("osio_usage_count", 0),
"latest_version": latest_version,
"recommended_latest_version": recommended_latest_version,
"github": github_details,
"code_metrics": code_metrics
}
# Add transitive block for transitive deps
if component.get('transitive', {}):
if not cves:
return None
else:
component_summary['transitive'] = component.get('transitive')
return component_summary
def filter_versions(epv_list, input_stack):
"""First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date"""
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
# needed for maven version like 1.5.2.RELEASE to be converted to
# 1.5.2-RELEASE for semantic version to work'
semversion = version.replace('.', '-', 3)
semversion = semversion.replace('-', '.', 2)
if name and version:
# Select Latest Version and add to filter_list if
# latest version is > current version
latest_version = select_latest_version(
epv.get('pkg').get('libio_latest_version', [''])[0],
epv.get('pkg').get('latest_version', [''])[0])
if latest_version and latest_version == version:
try:
if sv.SpecItem('>=' +
input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['latest_version'] = latest_version
new_dict[name]['latest_version'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
# Check for Dependency Count Attribute. Add Max deps count version
# if version > current version
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get('deps_count', {}).get(
'deps_count', 0):
try:
if sv.SpecItem(
'>=' +
input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['deps_count'] = {
"version": version,
"deps_count": deps_count
}
new_dict[name]['deps_count'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
# Check for github release date. Add version with most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > \
pkg_dict[name].get('gh_release_date', {}).get('gh_release_date',
0):
try:
if sv.SpecItem(
'>=' +
input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['gh_release_date'] = {
"version": version,
"gh_release_date": gh_release_date
}
new_dict[name]['gh_release_date'] = epv.get(
'ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
new_list = []
for package, contents in new_dict.items():
if 'latest_version' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['latest_version']
})
elif 'deps_count' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['deps_count']
})
elif 'gh_release_date' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['gh_release_date']
})
return new_list, filtered_comp_list
