def find_latest_version(): """Find the latest version.""" tmp_lst = [] for repo in REPO_DATA: del tmp_lst[:] FINAL_DATA[repo] = {} FINAL_DATA[repo]['notify'] = "false" latest_ver = '' libio_ver = '' deps = REPO_DATA[repo]['dependencies'] for dep in deps: tmp_json = {} pkg = VERSION_DATA[dep]['package'] if pkg in PACKAGE_DATA and \ (dep in NEW_VERSION_DATA or dep in VERSION_DATA): if 'latest' in PACKAGE_DATA[pkg]: latest_ver = PACKAGE_DATA[pkg]['latest'] if 'libio' in PACKAGE_DATA[pkg]: libio_ver = PACKAGE_DATA[pkg]['libio'] cur_ver = VERSION_DATA[dep]['version'] pkg_name = PACKAGE_DATA[pkg]['name'] latest_version = \ select_latest_version(cur_ver, libio_ver, latest_ver, pkg_name) if latest_version != cur_ver: FINAL_DATA[repo]['notify'] = "true" tmp_json['ecosystem'] = REPO_DATA[repo]['ecosystem'] tmp_json['name'] = pkg_name tmp_json['version'] = cur_ver tmp_json['latest_version'] = latest_version tmp_lst.append(tmp_json) FINAL_DATA[repo]['version_updates'] = tmp_lst[:]
def test_select_latest_version(): """Test the function select_latest_version.""" lat_ver = select_latest_version("-1", ["3.4.5", "3.4.1"], "pkg") assert lat_ver is "3.4.5" lat_ver = select_latest_version("-1", ["3.4.1", "3.4.5"], "pkg") assert lat_ver is "3.4.5" lat_ver = select_latest_version("3.4.5", ["3.4.1", "3.4.0"], "pkg") assert lat_ver is "3.4.5" lat_ver = select_latest_version("-1", ["-1", "-1"], "pkg") assert lat_ver is '' lat_ver = select_latest_version(["abc"], [{"a": "b"}, ["b"]], "pkg") assert lat_ver is ''
def get_recommended_version(ecosystem, name, version): """Fetch the recommended version in case of CVEs.""" query = "g.V().has('ecosystem', '{eco}').has('name', '{pkg}')" \ ".out('has_version').not(out('has_cve')).values('version');"\ .format(eco=ecosystem, pkg=name) payload = {'gremlin': query} result = execute_gremlin_dsl(url=GREMLIN_SERVER_URL_REST, payload=payload) if result: versions = result['result']['data'] if len(versions) == 0: return None else: return None rec_version = version for ver in versions: rec_version = select_latest_version(ver, rec_version) if rec_version == version: return None return rec_version
def filter_versions(epv_list, input_stack, external_request_id=None, rec_type=None): """Filter the EPVs according to following rules. First filter fetches only EPVs that 1. has No CVEs 2. are Present in Graph Apply additional filter based on following. That is sorted based on 3. Latest Version 4. Dependents Count in Github Manifest Data 5. Github Release Date """ # TODO: reduce cyclomatic complexity logger.info("Filtering {} for external_request_id {}".format( rec_type, external_request_id)) pkg_dict = defaultdict(dict) new_dict = defaultdict(dict) filtered_comp_list = [] # TODO: refactoring for epv in epv_list: name = epv.get('pkg', {}).get('name', [''])[0] version = epv.get('ver', {}).get('version', [''])[0] # needed for maven version like 1.5.2.RELEASE to be converted to # 1.5.2-RELEASE for semantic version to work' semversion_tuple = version_info_tuple( convert_version_to_proper_semantic(version, name)) input_stack_tuple = version_info_tuple( convert_version_to_proper_semantic(input_stack.get(name, ''), name)) if name and version: # Select Latest Version and add to filter_list if # latest version is > current version latest_version = select_latest_version( version, epv.get('pkg').get('libio_latest_version', [''])[0], epv.get('pkg').get('latest_version', [''])[0], name) if latest_version and latest_version == version: try: if semversion_tuple >= input_stack_tuple: pkg_dict[name]['latest_version'] = latest_version new_dict[name]['latest_version'] = epv.get('ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: logger.exception( "Unexpected ValueError while filtering latest version!" ) pass # Check for Dependency Count Attribute. Add Max deps count version # if version > current version deps_count = epv.get('ver').get('dependents_count', [-1])[0] if deps_count > 0: if 'deps_count' not in pkg_dict[name] or \ deps_count > pkg_dict[name].get('deps_count', {}).get( 'deps_count', 0): try: if semversion_tuple >= input_stack_tuple: pkg_dict[name]['deps_count'] = { "version": version, "deps_count": deps_count } new_dict[name]['deps_count'] = epv.get('ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: logger.exception( "Unexpected ValueError while filtering dependency count!" ) pass # Check for github release date. Add version with most recent github release date gh_release_date = epv.get('ver').get('gh_release_date', [0])[0] if gh_release_date > 0.0: if 'gh_release_date' not in pkg_dict[name] or \ gh_release_date > \ pkg_dict[name].get('gh_release_date', {}).get('gh_release_date', 0): try: if semversion_tuple >= input_stack_tuple: pkg_dict[name]['gh_release_date'] = { "version": version, "gh_release_date": gh_release_date } new_dict[name]['gh_release_date'] = epv.get( 'ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: logger.exception( "Unexpected ValueError while filtering github release date!" ) pass logger.info( "Data Dict new_dict for external_request_id {} is {}".format( external_request_id, new_dict)) logger.info( "Data List filtered_comp_list for external_request_id {} is {}". format(external_request_id, filtered_comp_list)) # TODO: refactoring new_list = [] for package, contents in new_dict.items(): if 'latest_version' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['latest_version'] }) elif 'deps_count' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['deps_count'] }) elif 'gh_release_date' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['gh_release_date'] }) return new_list, filtered_comp_list
def filter_versions(epv_list, input_stack, external_request_id=None, rec_type=None): """Filter the EPVs according to following rules. First filter fetches only EPVs that 1. has No CVEs 2. are Present in Graph Apply additional filter based on following. That is sorted based on 3. Latest Version 4. Dependents Count in Github Manifest Data 5. Github Release Date """ logger.info("Filtering {} for external_request_id {}".format(rec_type, external_request_id)) pkg_dict = defaultdict(dict) new_dict = defaultdict(dict) filtered_comp_list = [] for epv in epv_list: name = epv.get('pkg', {}).get('name', [''])[0] version = epv.get('ver', {}).get('version', [''])[0] libio_latest_version = epv.get('pkg').get('libio_latest_version', [''])[0] latest_version = epv.get('pkg').get('latest_version', [''])[0] # Convert version to a proper semantic case semversion_tuple = version_info_tuple( convert_version_to_proper_semantic(version, name)) input_stack_tuple = version_info_tuple( convert_version_to_proper_semantic(input_stack.get(name, ''), name)) if name and version: # Select highest version based on input or graph as latest version latest_version = select_latest_version( version, libio_latest_version, latest_version, name ) if latest_version and latest_version == version: pkg_dict, new_dict, filtered_comp_list = GraphDB.add_version_to_filtered_list( epv=epv, key='latest_version', val=latest_version, pkg_dict=pkg_dict, new_dict=new_dict, filtered_comp_list=filtered_comp_list, semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple) # Select Version based on highest dependents count (usage) deps_count = epv.get('ver').get('dependents_count', [-1])[0] if deps_count > 0: if 'deps_count' not in pkg_dict[name] or \ deps_count > pkg_dict[name].get['deps_count'].get('deps_count', 0): pkg_dict, new_dict, filtered_comp_list = \ GraphDB.add_version_to_filtered_list( epv=epv, key='deps_count', val=deps_count, pkg_dict=pkg_dict, new_dict=new_dict, filtered_comp_list=filtered_comp_list, semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple) # Select Version with the most recent github release date gh_release_date = epv.get('ver').get('gh_release_date', [0.0])[0] if gh_release_date > 0.0: if 'gh_release_date' not in pkg_dict[name] or \ gh_release_date > pkg_dict[name]['gh_release_date'].\ get('gh_release_date', 0.0): pkg_dict, new_dict, filtered_comp_list = \ GraphDB.add_version_to_filtered_list( epv=epv, key='gh_release_date', val=gh_release_date, pkg_dict=pkg_dict, new_dict=new_dict, filtered_comp_list=filtered_comp_list, semversion_tuple=semversion_tuple, input_stack_tuple=input_stack_tuple) logger.info("Data Dict new_dict for external_request_id {} is {}".format( external_request_id, new_dict)) logger.info("Data List filtered_comp_list for external_request_id {} is {}".format( external_request_id, filtered_comp_list)) new_list = GraphDB.prepare_final_filtered_list(new_dict) return new_list, filtered_comp_list
def find_latest_version(pkg_data, version_data, new_version_data, transitive_flag="false"): """Find the latest version.""" print("find_latest_version() started") tr_list = [] dir_list = [] for repo in REPO_DATA: del tr_list[:] del dir_list[:] if transitive_flag is "false": deps = REPO_DATA[repo]['dependencies'] FINAL_DATA[repo] = { 'notify': 'false', 'transitive_updates': [], 'direct_updates': [] } else: if 'tr_dependencies' in REPO_DATA[repo]: deps = REPO_DATA[repo]['tr_dependencies'] else: deps = [] for dep in deps: tmp_json = {} ver_list = [] pkg = version_data[dep]['package'] if pkg in pkg_data and \ (dep in new_version_data or dep in version_data) and \ (transitive_flag is "false" or ( FINAL_DATA[repo]['notify'] is "true" and transitive_flag is "true")): if 'versions' in pkg_data[pkg]: versions = pkg_data[pkg]['versions'] for ver in versions: ver_list.append(new_version_data[ver]['version']) cur_ver = version_data[dep]['version'] pkg_name = pkg_data[pkg]['name'] latest_version = \ select_latest_version(cur_ver, ver_list, pkg_name) eco = REPO_DATA[repo]['ecosystem'] latest_key = eco + ":" + pkg_name + ":" + latest_version if latest_version != cur_ver and \ (latest_key in new_version_data or latest_key in version_data): FINAL_DATA[repo]['notify'] = "true" tmp_json = { 'ecosystem': eco, 'name': pkg_name, 'version': cur_ver, 'latest_version': latest_version } if transitive_flag is "true": tr_list.append(tmp_json) else: dir_list.append(tmp_json) if tr_list: FINAL_DATA[repo]['transitive_updates'].extend(tr_list[:]) if dir_list: FINAL_DATA[repo]['direct_updates'].extend(dir_list[:]) print("find_latest_version() ended")
def extract_component_details(component): github_details = { "dependent_projects": component.get("package", {}).get("libio_dependents_projects", [-1])[0], "dependent_repos": component.get("package", {}).get("libio_dependents_repos", [-1])[0], "total_releases": component.get("package", {}).get("libio_total_releases", [-1])[0], "latest_release_duration": str( datetime.datetime.fromtimestamp( component.get("package", {}).get("libio_latest_release", [1496302486.0])[0])), "first_release_date": "Apr 16, 2010", "issues": { "month": { "opened": component.get("package", {}).get("gh_issues_last_month_opened", [-1])[0], "closed": component.get("package", {}).get("gh_issues_last_month_closed", [-1])[0] }, "year": { "opened": component.get("package", {}).get("gh_issues_last_year_opened", [-1])[0], "closed": component.get("package", {}).get("gh_issues_last_year_closed", [-1])[0] } }, "pull_requests": { "month": { "opened": component.get("package", {}).get("gh_prs_last_month_opened", [-1])[0], "closed": component.get("package", {}).get("gh_prs_last_month_closed", [-1])[0] }, "year": { "opened": component.get("package", {}).get("gh_prs_last_year_opened", [-1])[0], "closed": component.get("package", {}).get("gh_prs_last_year_closed", [-1])[0] } }, "stargazers_count": component.get("package", {}).get("gh_stargazers", [-1])[0], "forks_count": component.get("package", {}).get("gh_forks", [-1])[0], "open_issues_count": component.get("package", {}).get("gh_open_issues_count", [-1])[0], "contributors": component.get("package", {}).get("gh_contributors_count", [-1])[0], "size": "N/A" } used_by = component.get("package", {}).get("libio_usedby", []) used_by_list = [] for epvs in used_by: slc = epvs.split(':') used_by_dict = {'name': slc[0], 'stars': int(slc[1])} used_by_list.append(used_by_dict) github_details['used_by'] = used_by_list code_metrics = { "code_lines": component.get("version", {}).get("cm_loc", [-1])[0], "average_cyclomatic_complexity": component.get("version", {}).get("cm_avg_cyclomatic_complexity", [-1])[0], "total_files": component.get("version", {}).get("cm_num_files", [-1])[0] } cves = [] for cve in component.get("version", {}).get("cve_ids", []): component_cve = {'CVE': cve.split(':')[0], 'CVSS': cve.split(':')[1]} cves.append(component_cve) licenses = component.get("version", {}).get("declared_licenses", []) name = component.get("version", {}).get("pname", [""])[0] version = component.get("version", {}).get("version", [""])[0] ecosystem = component.get("version", {}).get("pecosystem", [""])[0] latest_version = select_latest_version( version, component.get("package", {}).get("libio_latest_version", [""])[0], component.get("package", {}).get("latest_version", [""])[0]) component_summary = { "ecosystem": ecosystem, "name": name, "version": version, "licenses": licenses, "security": cves, "osio_user_count": component.get("version", {}).get("osio_usage_count", 0), "latest_version": latest_version, "github": github_details, "code_metrics": code_metrics } return component_summary
def extract_component_details(component): """Extract details from given component.""" date = format_date( component.get("package", {}).get("gh_refreshed_on", ["N/A"])[0]) github_details = { "dependent_projects": component.get("package", {}).get("libio_dependents_projects", [-1])[0], "dependent_repos": component.get("package", {}).get("libio_dependents_repos", [-1])[0], "total_releases": component.get("package", {}).get("libio_total_releases", [-1])[0], "latest_release_duration": str( datetime.datetime.fromtimestamp( component.get("package", {}).get("libio_latest_release", [1496302486.0])[0])), "first_release_date": "Apr 16, 2010", "issues": { "month": { "opened": component.get("package", {}).get("gh_issues_last_month_opened", [-1])[0], "closed": component.get("package", {}).get("gh_issues_last_month_closed", [-1])[0] }, "year": { "opened": component.get("package", {}).get("gh_issues_last_year_opened", [-1])[0], "closed": component.get("package", {}).get("gh_issues_last_year_closed", [-1])[0] } }, "pull_requests": { "month": { "opened": component.get("package", {}).get("gh_prs_last_month_opened", [-1])[0], "closed": component.get("package", {}).get("gh_prs_last_month_closed", [-1])[0] }, "year": { "opened": component.get("package", {}).get("gh_prs_last_year_opened", [-1])[0], "closed": component.get("package", {}).get("gh_prs_last_year_closed", [-1])[0] } }, "stargazers_count": component.get("package", {}).get("gh_stargazers", [-1])[0], "forks_count": component.get("package", {}).get("gh_forks", [-1])[0], "refreshed_on": date, "open_issues_count": component.get("package", {}).get("gh_open_issues_count", [-1])[0], "contributors": component.get("package", {}).get("gh_contributors_count", [-1])[0], "size": "N/A" } used_by = component.get("package", {}).get("libio_usedby", []) used_by_list = [] for epvs in used_by: slc = epvs.split(':') used_by_dict = {'name': slc[0], 'stars': int(slc[1])} used_by_list.append(used_by_dict) github_details['used_by'] = used_by_list code_metrics = { "code_lines": component.get("version", {}).get("cm_loc", [-1])[0], "average_cyclomatic_complexity": component.get("version", {}).get("cm_avg_cyclomatic_complexity", [-1])[0], "total_files": component.get("version", {}).get("cm_num_files", [-1])[0] } cves = [] recommended_latest_version = None name = component.get("version", {}).get("pname", [""])[0] version = component.get("version", {}).get("version", [""])[0] ecosystem = component.get("version", {}).get("pecosystem", [""])[0] if len(component.get("cves", [])) > 0: for cve in component.get("cves", []): component_cve = { 'CVE': cve.get('cve_id')[0], 'CVSS': cve.get('cvss_v2', [''])[0] } cves.append(component_cve) recommended_latest_version = component.get("package", {}).get( "latest_non_cve_version", "") if not recommended_latest_version: recommended_latest_version = get_recommended_version( ecosystem, name, version) licenses = component.get("version", {}).get("declared_licenses", []) latest_version = select_latest_version( version, component.get("package", {}).get("libio_latest_version", [""])[0], component.get("package", {}).get("latest_version", [""])[0], name) component_summary = { "ecosystem": ecosystem, "name": name, "version": version, "licenses": licenses, "security": cves, "osio_user_count": component.get("version", {}).get("osio_usage_count", 0), "latest_version": latest_version, "recommended_latest_version": recommended_latest_version, "github": github_details, "code_metrics": code_metrics } # Add transitive block for transitive deps if component.get('transitive', {}): if not cves: return None else: component_summary['transitive'] = component.get('transitive') return component_summary
def filter_versions(epv_list, input_stack): """First filter fetches only EPVs that 1. has No CVEs 2. are Present in Graph Apply additional filter based on following. That is sorted based on 3. Latest Version 4. Dependents Count in Github Manifest Data 5. Github Release Date""" pkg_dict = defaultdict(dict) new_dict = defaultdict(dict) filtered_comp_list = [] for epv in epv_list: name = epv.get('pkg', {}).get('name', [''])[0] version = epv.get('ver', {}).get('version', [''])[0] # needed for maven version like 1.5.2.RELEASE to be converted to # 1.5.2-RELEASE for semantic version to work' semversion = version.replace('.', '-', 3) semversion = semversion.replace('-', '.', 2) if name and version: # Select Latest Version and add to filter_list if # latest version is > current version latest_version = select_latest_version( epv.get('pkg').get('libio_latest_version', [''])[0], epv.get('pkg').get('latest_version', [''])[0]) if latest_version and latest_version == version: try: if sv.SpecItem('>=' + input_stack.get(name, '0.0.0')).match( sv.Version(semversion)): pkg_dict[name]['latest_version'] = latest_version new_dict[name]['latest_version'] = epv.get('ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: pass # Check for Dependency Count Attribute. Add Max deps count version # if version > current version deps_count = epv.get('ver').get('dependents_count', [-1])[0] if deps_count > 0: if 'deps_count' not in pkg_dict[name] or \ deps_count > pkg_dict[name].get('deps_count', {}).get( 'deps_count', 0): try: if sv.SpecItem( '>=' + input_stack.get(name, '0.0.0')).match( sv.Version(semversion)): pkg_dict[name]['deps_count'] = { "version": version, "deps_count": deps_count } new_dict[name]['deps_count'] = epv.get('ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: pass # Check for github release date. Add version with most recent github release date gh_release_date = epv.get('ver').get('gh_release_date', [0])[0] if gh_release_date > 0.0: if 'gh_release_date' not in pkg_dict[name] or \ gh_release_date > \ pkg_dict[name].get('gh_release_date', {}).get('gh_release_date', 0): try: if sv.SpecItem( '>=' + input_stack.get(name, '0.0.0')).match( sv.Version(semversion)): pkg_dict[name]['gh_release_date'] = { "version": version, "gh_release_date": gh_release_date } new_dict[name]['gh_release_date'] = epv.get( 'ver') new_dict[name]['pkg'] = epv.get('pkg') filtered_comp_list.append(name) except ValueError: pass new_list = [] for package, contents in new_dict.items(): if 'latest_version' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['latest_version'] }) elif 'deps_count' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['deps_count'] }) elif 'gh_release_date' in contents: new_list.append({ "pkg": contents['pkg'], "ver": contents['gh_release_date'] }) return new_list, filtered_comp_list