def filter_versions(epv_list, input_stack, external_request_id=None, rec_type=None):
"""Filter the EPVs according to following rules.
First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date
"""
logger.info("Filtering {} for external_request_id {}".format(rec_type, external_request_id))
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
libio_latest_version = epv.get('pkg').get('libio_latest_version', [''])[0]
latest_version = epv.get('pkg').get('latest_version', [''])[0]
# Convert version to a proper semantic case
semversion_tuple = version_info_tuple(
convert_version_to_proper_semantic(version, name))
input_stack_tuple = version_info_tuple(
convert_version_to_proper_semantic(input_stack.get(name, ''), name))
if name and version:
# Select highest version based on input or graph as latest version
latest_version = select_latest_version(
version, libio_latest_version, latest_version, name
)
if latest_version and latest_version == version:
pkg_dict, new_dict, filtered_comp_list = GraphDB.add_version_to_filtered_list(
epv=epv, key='latest_version', val=latest_version, pkg_dict=pkg_dict,
new_dict=new_dict, filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple)
# Select Version based on highest dependents count (usage)
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get['deps_count'].get('deps_count', 0):
pkg_dict, new_dict, filtered_comp_list = \
GraphDB.add_version_to_filtered_list(
epv=epv, key='deps_count', val=deps_count, pkg_dict=pkg_dict,
new_dict=new_dict, filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple,
input_stack_tuple=input_stack_tuple)
# Select Version with the most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0.0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > pkg_dict[name]['gh_release_date'].\
get('gh_release_date', 0.0):
pkg_dict, new_dict, filtered_comp_list = \
GraphDB.add_version_to_filtered_list(
epv=epv, key='gh_release_date', val=gh_release_date,
pkg_dict=pkg_dict, new_dict=new_dict,
filtered_comp_list=filtered_comp_list,
semversion_tuple=semversion_tuple,
input_stack_tuple=input_stack_tuple)
logger.info("Data Dict new_dict for external_request_id {} is {}".format(
external_request_id, new_dict))
logger.info("Data List filtered_comp_list for external_request_id {} is {}".format(
external_request_id, filtered_comp_list))
new_list = GraphDB.prepare_final_filtered_list(new_dict)
return new_list, filtered_comp_list
def filter_versions(epv_list,
input_stack,
external_request_id=None,
rec_type=None):
"""Filter the EPVs according to following rules.
First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date
"""
# TODO: reduce cyclomatic complexity
logger.info("Filtering {} for external_request_id {}".format(
rec_type, external_request_id))
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
# TODO: refactoring
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
# needed for maven version like 1.5.2.RELEASE to be converted to
# 1.5.2-RELEASE for semantic version to work'
semversion_tuple = version_info_tuple(
convert_version_to_proper_semantic(version, name))
input_stack_tuple = version_info_tuple(
convert_version_to_proper_semantic(input_stack.get(name, ''),
name))
if name and version:
# Select Latest Version and add to filter_list if
# latest version is > current version
latest_version = select_latest_version(
version,
epv.get('pkg').get('libio_latest_version', [''])[0],
epv.get('pkg').get('latest_version', [''])[0], name)
if latest_version and latest_version == version:
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['latest_version'] = latest_version
new_dict[name]['latest_version'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering latest version!"
)
pass
# Check for Dependency Count Attribute. Add Max deps count version
# if version > current version
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get('deps_count', {}).get(
'deps_count', 0):
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['deps_count'] = {
"version": version,
"deps_count": deps_count
}
new_dict[name]['deps_count'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering dependency count!"
)
pass
# Check for github release date. Add version with most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > \
pkg_dict[name].get('gh_release_date', {}).get('gh_release_date',
0):
try:
if semversion_tuple >= input_stack_tuple:
pkg_dict[name]['gh_release_date'] = {
"version": version,
"gh_release_date": gh_release_date
}
new_dict[name]['gh_release_date'] = epv.get(
'ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
logger.exception(
"Unexpected ValueError while filtering github release date!"
)
pass
logger.info(
"Data Dict new_dict for external_request_id {} is {}".format(
external_request_id, new_dict))
logger.info(
"Data List filtered_comp_list for external_request_id {} is {}".
format(external_request_id, filtered_comp_list))
# TODO: refactoring
new_list = []
for package, contents in new_dict.items():
if 'latest_version' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['latest_version']
})
elif 'deps_count' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['deps_count']
})
elif 'gh_release_date' in contents:
new_list.append({
"pkg": contents['pkg'],
"ver": contents['gh_release_date']
})
return new_list, filtered_comp_list
def filter_versions(epv_list, input_stack):
"""First filter fetches only EPVs that
1. has No CVEs
2. are Present in Graph
Apply additional filter based on following. That is sorted based on
3. Latest Version
4. Dependents Count in Github Manifest Data
5. Github Release Date"""
pkg_dict = defaultdict(dict)
new_dict = defaultdict(dict)
filtered_comp_list = []
for epv in epv_list:
name = epv.get('pkg', {}).get('name', [''])[0]
version = epv.get('ver', {}).get('version', [''])[0]
# needed for maven version like 1.5.2.RELEASE to be converted to
# 1.5.2-RELEASE for semantic version to work'
semversion = convert_version_to_proper_semantic(version)
if name and version:
# Select Latest Version and add to filter_list if
# latest version is > current version
latest_version = select_latest_version(
version,
epv.get('pkg').get('libio_latest_version', [''])[0],
epv.get('pkg').get('latest_version', [''])[0]
)
if latest_version and latest_version == version:
try:
if sv.SpecItem('>=' + input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['latest_version'] = latest_version
new_dict[name]['latest_version'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
# Check for Dependency Count Attribute. Add Max deps count version
# if version > current version
deps_count = epv.get('ver').get('dependents_count', [-1])[0]
if deps_count > 0:
if 'deps_count' not in pkg_dict[name] or \
deps_count > pkg_dict[name].get('deps_count', {}).get(
'deps_count', 0):
try:
if sv.SpecItem('>=' + input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['deps_count'] = {"version": version,
"deps_count": deps_count}
new_dict[name]['deps_count'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
# Check for github release date. Add version with most recent github release date
gh_release_date = epv.get('ver').get('gh_release_date', [0])[0]
if gh_release_date > 0.0:
if 'gh_release_date' not in pkg_dict[name] or \
gh_release_date > \
pkg_dict[name].get('gh_release_date', {}).get('gh_release_date',
0):
try:
if sv.SpecItem('>=' + input_stack.get(name, '0.0.0')).match(
sv.Version(semversion)):
pkg_dict[name]['gh_release_date'] = {
"version": version,
"gh_release_date": gh_release_date}
new_dict[name]['gh_release_date'] = epv.get('ver')
new_dict[name]['pkg'] = epv.get('pkg')
filtered_comp_list.append(name)
except ValueError:
pass
new_list = []
for package, contents in new_dict.items():
if 'latest_version' in contents:
new_list.append({"pkg": contents['pkg'], "ver": contents['latest_version']})
elif 'deps_count' in contents:
new_list.append({"pkg": contents['pkg'], "ver": contents['deps_count']})
elif 'gh_release_date' in contents:
new_list.append({"pkg": contents['pkg'], "ver": contents['gh_release_date']})
return new_list, filtered_comp_list
