def hook_preconvert_02_submitters(): for filename in os.listdir('input/submitters'): if not filename.endswith('.json'): continue with open('input/submitters/' + filename, 'r') as f: print("processing: " + filename) submitter = Submitter(json.load(f)) submitters[submitter.ID] = submitter set_latex_value('NumSubmitters', len(submitters))
def calculate_latencies(version_dates): linux_latencies = latency(version_dates['linux'], OrderedDict(avo.os_to_kernel)) set_latex_value('linuxMeanUpdateLatency', ufloat(statistics.mean(linux_latencies.values()),statistics.stdev(linux_latencies.values()))) openssl_latencies = latency(version_dates['openssl'], OrderedDict(avo.os_to_project['openssl'])) set_latex_value('opensslMeanUpdateLatency', ufloat(statistics.mean(openssl_latencies.values()),statistics.stdev(openssl_latencies.values()))) bouncycastle_latencies = latency(version_dates['bouncycastle'], OrderedDict(avo.os_to_project['bouncycastle'])) set_latex_value('bouncycastleMeanUpdateLatency',ufloat(statistics.mean(bouncycastle_latencies.values()),statistics.stdev(bouncycastle_latencies.values())))
def hook_preconvert_external_linecount(): global python_export_file_contents, total_lines with open('input/external_lines_of_code.json') as f: rjson = json.load(f) for project, lines in rjson.items(): if len(project) > 0 and len(lines) > 0: project_lines[project] = int(lines) sorted_pl = sorted(project_lines.items(), key=lambda x : x[1])#Sort by lines of code total_lines = sum(project_lines.values()) set_latex_value('TotalExternalLines', num2word(total_lines)) set_latex_value('NumExternalProjects', len(sorted_pl)) big_total_lines = sum(map(lambda x : x[1], sorted_pl[40:]))#TODO factor this 40 out set_latex_value('NumBigExternalLinesOfCode', num2word(big_total_lines)) set_latex_value('BigExternalLinesOfCodePerc', big_total_lines/total_lines, t='perc') python_export_file_contents += '\ntotal_external_lines = ' + str(total_lines) + '\n' python_export_file_contents += '\nexternal_project_lines = ' + str(sorted_pl) + '\n'
def count_versions(data): total = 0 totals = [] for project, values in data.items(): values_set = set(map(lambda x : x[1], values.items())) num_values = len(values_set) total += num_values totals.append(num_values) totals = sorted(totals) set_latex_value('BigExternalMedianVersions', numpy.median(numpy.array(totals))) set_latex_value('BigExternalMeanVersions', ufloat(numpy.mean(numpy.array(totals)), numpy.std(numpy.array(totals)))) set_latex_value('BigExternalTotalVersions', total)
def hook_preconvert_tag_versions(): global python_export_file_contents #upstreams = ['openssl', 'bouncycastle', 'libogg', 'libxml2', 'openssh'] upstreams = [ 'aac', 'kernel-headers', 'bouncycastle', 'sonivox', 'tcpdump', 'freetype', 'libnfc-nxp', 'srec', 'elfutils', 'apache-xml', 'openssh', 'stlport', 'linux-tools-perf', 'e2fsprogs', 'apache-harmony', 'eigen', 'jmonkeyengine', 'protobuf', 'opencv', 'guava', 'libxml2', 'bluetooth', 'sqlite', 'antlr', 'bison', 'libvpx', 'wpa_supplicant_8', 'compiler-rt', 'libcxx', 'skia', 'openssl', 'qemu', 'vixl', 'icu', 'valgrind', 'mesa3d', 'llvm', 'clang', 'chromium', 'chromium_org'] set_latex_value('NumBigExternalProjects',len(upstreams)) existing_upstreams = upstreams[ :] # May need to remove ones for which we lack data data = dict() for upstream in upstreams: tag_versions(upstream, existing_upstreams, data) set_latex_value('NumAnalysedExternalProjects', len(existing_upstreams)) analysed_lines_of_code = 0 for upstream in existing_upstreams: analysed_lines_of_code += project_lines[upstream] set_latex_value('NumAnalysedExternalLinesOfCode', num2word(analysed_lines_of_code)) set_latex_value('AnalysedExternalLinesOfCodePerc', analysed_lines_of_code/total_lines, t='perc') count_versions(data) python_export_file_contents += '\nupstreams = ' + str(existing_upstreams) + '\n' python_export_file_contents += '\nos_to_project = ' + str(data) + '\n'
def update_versions(): targz = re.compile(r'.*\.tar\.gz.*') openssl_lines = get_lines('https://www.openssl.org/source/', targz) openssl_lines.extend(get_lines('https://www.openssl.org/source/old/0.9.x/', targz)) openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.0/', targz)) openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.1/', targz)) openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.2/', targz)) openssl_version_dates = get_version_dates(openssl_lines, re.compile(r'.*"openssl-([^"]+)\.tar\.gz".*'),re.compile(r'.*?\d+ ([^<]+) <.*')) set_latex_value('opensslNumVersions', num_in_range(openssl_version_dates)) save_to_csv('openssl', openssl_version_dates) linux_lines = get_lines('https://www.kernel.org/pub/linux/kernel/v2.6/', targz) linux_lines.extend(get_lines('https://www.kernel.org/pub/linux/kernel/v3.0/', targz)) linux_version_dates = get_version_dates(linux_lines, re.compile(r'.*"linux-([0-9.]+)\.tar\.gz".*'), re.compile('.*?\s+\s([0-9\-A-z: ]+)\s+\d+M.*')) set_latex_value('linuxNumVersions', num_in_range(linux_version_dates)) save_to_csv('linux', linux_version_dates) bouncycastle_version_dates = fetch_bouncy_castle() set_latex_value('bouncycastleNumVersions', num_in_range(bouncycastle_version_dates)) save_to_csv('bouncycastle', bouncycastle_version_dates) return {'openssl': OrderedDict(openssl_version_dates), 'linux' : OrderedDict(linux_version_dates), 'bouncycastle': OrderedDict(bouncycastle_version_dates)}
def hook_preconvert_stats(): set_latex_value('NumVulnerabilities', len(vulnerabilities)) num_vuln_all_android = 0 num_vuln_specific = 0 first_submission = None last_submission = None first_date = None last_date = None for vuln in vulnerabilities: manufacturers = vuln.manufacturers() if 'all' in [x[0] for x in manufacturers]: num_vuln_all_android += 1 else: num_vuln_specific += 1 for submission in vuln.submissions(): on = submission.on if first_submission == None: first_submission = on last_submission = on else: if on < first_submission: first_submission = on elif on > last_submission: last_submission = on first = vuln.first_date() last = vuln.last_date() if first_date == None: first_date = first last_date = last else: if first < first_date: first_date = first if last > last_date: last_date = last set_latex_value('NumVulnAllAndroid', num_vuln_all_android) set_latex_value('NumVulnSpecific', num_vuln_specific) set_latex_value('StartDate', first_submission) set_latex_value('EndDate', last_submission) set_latex_value('FirstDataDate', first_date) set_latex_value('LastDataDate', last_date) set_latex_value('VulnsPerYear', (ufloat(len(vulnerabilities),sqrt(len(vulnerabilities)))/((last_date - first_date)/datetime.timedelta(1)))*365) set_latex_value('VulnsPerYearAllAndroid', (ufloat(num_vuln_all_android,sqrt(num_vuln_all_android))/((last_date - first_date)/datetime.timedelta(1)))*365) set_latex_value('VulnsPerYearTwosf', (ufloat(len(vulnerabilities),sqrt(len(vulnerabilities)))/((last_date - first_date)/datetime.timedelta(1)))*365, sig_figs=2) set_latex_value('VulnsPerYearAllAndroidTwosf', (ufloat(num_vuln_all_android,sqrt(num_vuln_all_android))/((last_date - first_date)/datetime.timedelta(1)))*365, sig_figs=2) vuln_table = r'\begin{table} \centering \begin{tabular}{l|l|c|c|p{19.5em}} Vulnerability & How known & Date & Categories & CVEs\\ \hline' for versions, date, name, how_known in raw_vulnerabilities: cves = vuln_by_name[name].cves() cvestring = ", ".join([r'{0}~\cite{{{0}}}'.format(x) if x.startswith('CVE') else x for x in cves]) vuln_table += r' \href{{http://androidvulnerabilities.org/vulnerabilities/{}}}{{{}}} & {} & {} & {} & {}\\'.format(name.replace(' ', '_'), try_shorten(name), how_known, date, ", ".join(vuln_by_name[name].categories()), cvestring) vuln_table += r'\end{tabular} \caption{Critical vulnerabilities in Android} \label{tab:andvulns} \end{table}' set_latex_value('TabAndVulns', vuln_table)