def hook_preconvert_02_submitters():
for filename in os.listdir('input/submitters'):
if not filename.endswith('.json'):
continue
with open('input/submitters/' + filename, 'r') as f:
print("processing: " + filename)
submitter = Submitter(json.load(f))
submitters[submitter.ID] = submitter
set_latex_value('NumSubmitters', len(submitters))
def calculate_latencies(version_dates):
linux_latencies = latency(version_dates['linux'], OrderedDict(avo.os_to_kernel))
set_latex_value('linuxMeanUpdateLatency', ufloat(statistics.mean(linux_latencies.values()),statistics.stdev(linux_latencies.values())))
openssl_latencies = latency(version_dates['openssl'], OrderedDict(avo.os_to_project['openssl']))
set_latex_value('opensslMeanUpdateLatency', ufloat(statistics.mean(openssl_latencies.values()),statistics.stdev(openssl_latencies.values())))
bouncycastle_latencies = latency(version_dates['bouncycastle'], OrderedDict(avo.os_to_project['bouncycastle']))
set_latex_value('bouncycastleMeanUpdateLatency',ufloat(statistics.mean(bouncycastle_latencies.values()),statistics.stdev(bouncycastle_latencies.values())))
def hook_preconvert_external_linecount():
global python_export_file_contents, total_lines
with open('input/external_lines_of_code.json') as f:
rjson = json.load(f)
for project, lines in rjson.items():
if len(project) > 0 and len(lines) > 0:
project_lines[project] = int(lines)
sorted_pl = sorted(project_lines.items(), key=lambda x : x[1])#Sort by lines of code
total_lines = sum(project_lines.values())
set_latex_value('TotalExternalLines', num2word(total_lines))
set_latex_value('NumExternalProjects', len(sorted_pl))
big_total_lines = sum(map(lambda x : x[1], sorted_pl[40:]))#TODO factor this 40 out
set_latex_value('NumBigExternalLinesOfCode', num2word(big_total_lines))
set_latex_value('BigExternalLinesOfCodePerc', big_total_lines/total_lines, t='perc')
python_export_file_contents += '\ntotal_external_lines = ' + str(total_lines) + '\n'
python_export_file_contents += '\nexternal_project_lines = ' + str(sorted_pl) + '\n'
def count_versions(data):
total = 0
totals = []
for project, values in data.items():
values_set = set(map(lambda x : x[1], values.items()))
num_values = len(values_set)
total += num_values
totals.append(num_values)
totals = sorted(totals)
set_latex_value('BigExternalMedianVersions', numpy.median(numpy.array(totals)))
set_latex_value('BigExternalMeanVersions', ufloat(numpy.mean(numpy.array(totals)), numpy.std(numpy.array(totals))))
set_latex_value('BigExternalTotalVersions', total)
def hook_preconvert_tag_versions():
global python_export_file_contents
#upstreams = ['openssl', 'bouncycastle', 'libogg', 'libxml2', 'openssh']
upstreams = [
'aac', 'kernel-headers', 'bouncycastle', 'sonivox', 'tcpdump', 'freetype', 'libnfc-nxp', 'srec', 'elfutils', 'apache-xml', 'openssh', 'stlport', 'linux-tools-perf', 'e2fsprogs', 'apache-harmony', 'eigen', 'jmonkeyengine',
'protobuf', 'opencv', 'guava', 'libxml2', 'bluetooth', 'sqlite', 'antlr', 'bison', 'libvpx', 'wpa_supplicant_8', 'compiler-rt', 'libcxx', 'skia', 'openssl', 'qemu', 'vixl', 'icu', 'valgrind', 'mesa3d', 'llvm', 'clang', 'chromium', 'chromium_org']
set_latex_value('NumBigExternalProjects',len(upstreams))
existing_upstreams = upstreams[
:] # May need to remove ones for which we lack data
data = dict()
for upstream in upstreams:
tag_versions(upstream, existing_upstreams, data)
set_latex_value('NumAnalysedExternalProjects', len(existing_upstreams))
analysed_lines_of_code = 0
for upstream in existing_upstreams:
analysed_lines_of_code += project_lines[upstream]
set_latex_value('NumAnalysedExternalLinesOfCode', num2word(analysed_lines_of_code))
set_latex_value('AnalysedExternalLinesOfCodePerc', analysed_lines_of_code/total_lines, t='perc')
count_versions(data)
python_export_file_contents += '\nupstreams = ' + str(existing_upstreams) + '\n'
python_export_file_contents += '\nos_to_project = ' + str(data) + '\n'
def update_versions():
targz = re.compile(r'.*\.tar\.gz.*')
openssl_lines = get_lines('https://www.openssl.org/source/', targz)
openssl_lines.extend(get_lines('https://www.openssl.org/source/old/0.9.x/', targz))
openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.0/', targz))
openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.1/', targz))
openssl_lines.extend(get_lines('https://www.openssl.org/source/old/1.0.2/', targz))
openssl_version_dates = get_version_dates(openssl_lines, re.compile(r'.*"openssl-([^"]+)\.tar\.gz".*'),re.compile(r'.*?\d+ ([^<]+) <.*'))
set_latex_value('opensslNumVersions', num_in_range(openssl_version_dates))
save_to_csv('openssl', openssl_version_dates)
linux_lines = get_lines('https://www.kernel.org/pub/linux/kernel/v2.6/', targz)
linux_lines.extend(get_lines('https://www.kernel.org/pub/linux/kernel/v3.0/', targz))
linux_version_dates = get_version_dates(linux_lines, re.compile(r'.*"linux-([0-9.]+)\.tar\.gz".*'), re.compile('.*?\s+\s([0-9\-A-z: ]+)\s+\d+M.*'))
set_latex_value('linuxNumVersions', num_in_range(linux_version_dates))
save_to_csv('linux', linux_version_dates)
bouncycastle_version_dates = fetch_bouncy_castle()
set_latex_value('bouncycastleNumVersions', num_in_range(bouncycastle_version_dates))
save_to_csv('bouncycastle', bouncycastle_version_dates)
return {'openssl': OrderedDict(openssl_version_dates), 'linux' : OrderedDict(linux_version_dates), 'bouncycastle': OrderedDict(bouncycastle_version_dates)}
def hook_preconvert_stats():
set_latex_value('NumVulnerabilities', len(vulnerabilities))
num_vuln_all_android = 0
num_vuln_specific = 0
first_submission = None
last_submission = None
first_date = None
last_date = None
for vuln in vulnerabilities:
manufacturers = vuln.manufacturers()
if 'all' in [x[0] for x in manufacturers]:
num_vuln_all_android += 1
else:
num_vuln_specific += 1
for submission in vuln.submissions():
on = submission.on
if first_submission == None:
first_submission = on
last_submission = on
else:
if on < first_submission:
first_submission = on
elif on > last_submission:
last_submission = on
first = vuln.first_date()
last = vuln.last_date()
if first_date == None:
first_date = first
last_date = last
else:
if first < first_date:
first_date = first
if last > last_date:
last_date = last
set_latex_value('NumVulnAllAndroid', num_vuln_all_android)
set_latex_value('NumVulnSpecific', num_vuln_specific)
set_latex_value('StartDate', first_submission)
set_latex_value('EndDate', last_submission)
set_latex_value('FirstDataDate', first_date)
set_latex_value('LastDataDate', last_date)
set_latex_value('VulnsPerYear', (ufloat(len(vulnerabilities),sqrt(len(vulnerabilities)))/((last_date - first_date)/datetime.timedelta(1)))*365)
set_latex_value('VulnsPerYearAllAndroid', (ufloat(num_vuln_all_android,sqrt(num_vuln_all_android))/((last_date - first_date)/datetime.timedelta(1)))*365)
set_latex_value('VulnsPerYearTwosf', (ufloat(len(vulnerabilities),sqrt(len(vulnerabilities)))/((last_date - first_date)/datetime.timedelta(1)))*365, sig_figs=2)
set_latex_value('VulnsPerYearAllAndroidTwosf', (ufloat(num_vuln_all_android,sqrt(num_vuln_all_android))/((last_date - first_date)/datetime.timedelta(1)))*365, sig_figs=2)
vuln_table = r'\begin{table} \centering \begin{tabular}{l|l|c|c|p{19.5em}} Vulnerability & How known & Date & Categories & CVEs\\ \hline'
for versions, date, name, how_known in raw_vulnerabilities:
cves = vuln_by_name[name].cves()
cvestring = ", ".join([r'{0}~\cite{{{0}}}'.format(x) if x.startswith('CVE') else x for x in cves])
vuln_table += r' \href{{http://androidvulnerabilities.org/vulnerabilities/{}}}{{{}}} & {} & {} & {} & {}\\'.format(name.replace(' ', '_'), try_shorten(name), how_known, date, ", ".join(vuln_by_name[name].categories()), cvestring)
vuln_table += r'\end{tabular} \caption{Critical vulnerabilities in Android} \label{tab:andvulns} \end{table}'
set_latex_value('TabAndVulns', vuln_table)