def main():
projects = load_projects_json()
total_projects = len(projects)
count = 0
bugless_count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id',
'JarMetadata.artifact_id',
'JarMetadata.version',
'JarMetadata.version_order',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type'
])
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
for d in doc_list:
bug_instances = d.get('BugCollection', {}).get('BugInstance', [])
if len(bug_instances) == 0:
bugless_count += 1
break
print '[%d:%d:%d] %s||%s: %d versions' % (
count, total_projects, bugless_count, p.group_id(),
p.artifact_id(), len(doc_list))
print "bugless: %d, total: %d" % (bugless_count, total)
def main():
projects = load_projects_json()
valid_projects = []
total = len(projects)
valid = 0
counter = 0
for p in projects:
counter += 1
key = '%s||%s' % (p.group_id(), p.artifact_id())
piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.version_order'])\
piter.evolution_list()
print '[%d:%d:%d] Checking ... %s' % (counter, valid, total, key),
if piter.valid():
valid_projects.append(key)
print ' ... Valid (%d versions)' % (len(piter.evolution_list()))
valid += 1
else:
print ' ... Invalid (%d versions)' % (len(piter.evolution_list()))
print 'Total: %d, Valid: %d' % (total, valid)
save_to_file('valid_projects.json', json.dumps(valid_projects))
def main():
projects = load_vuln_projects_json()
results = {}
security_bugs = [
'HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
]
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(
p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id', 'JarMetadata.artifact_id',
'JarMetadata.version', 'JarMetadata.jar_size',
'JarMetadata.version_order',
'JarMetadata.jar_last_modification_date',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type',
'BugCollection.BugInstance.Class.classname',
'BugCollection.BugInstance.priority'
])
doc_list = piter.documents_list()
documents = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
p.group_id(), p.artifact_id(),
len(doc_list))
for d in doc_list:
doc_results = {'JarMetadata': d['JarMetadata']}
doc_array_count = ArrayCount()
sec_instances = []
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
print bi
continue
bug_category = bi.get('category', '')
# method
if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
classnames = bi['Class']
classresults = []
if isinstance(classnames, list):
for c in classnames:
classresults.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
classresults.append(
classnames.get('classname', 'NotSet'))
sec_dict = {
'Category': bug_category,
'Type': bi.get('type', 'NotSet'),
'Priority': int(bi.get('priority', 0)),
'Class': classresults
}
sec_instances.append(sec_dict)
# counters
if bug_category == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
doc_array_count.incr('SECURITY_HIGH')
else:
doc_array_count.incr('SECURITY_LOW')
else:
doc_array_count.incr(bug_category)
#doc_array_count.incr(bug_category)
doc_results['Counters'] = doc_array_count.get_series()
doc_results['SecurityBugs'] = sec_instances
documents.append(doc_results)
key = '%s||%s' % (p.group_id(), p.artifact_id())
results[key] = {
'group_id': p.group_id(),
'artifact_id': p.artifact_id(),
'version_count': len(doc_list),
'versions': documents
}
#print results
save_to_file('project_counters.json', json.dumps(results))
def main():
projects = load_evolution_projects_json()
results = {}
security_bugs = [
'HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
]
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(
p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id', 'JarMetadata.artifact_id',
'JarMetadata.version', 'JarMetadata.version_order',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type',
'BugCollection.BugInstance.Class.classname',
'BugCollection.BugInstance.Method.name',
'BugCollection.BugInstance.Field.name'
])
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
p.group_id(), p.artifact_id(),
len(doc_list))
for d in doc_list:
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
#print 'Invalid BugInstance (%s)' % (bi,)
continue
bug_c = bi.get('category', '')
if bug_c == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
bug_category = 'SECURITY_HIGH'
else:
bug_category = 'SECURITY_LOW'
else:
bug_category = bug_c
# create signature
signatures_ids = []
classnames = bi['Class']
if isinstance(classnames, list):
for c in classnames:
signatures_ids.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
signatures_ids.append(classnames.get(
'classname', 'NotSet'))
# methods
methodnames = bi.get('Method', {})
if isinstance(methodnames, list):
for m in methodnames:
signatures_ids.append(m.get('name', 'NotSet'))
elif isinstance(methodnames, dict):
signatures_ids.append(methodnames.get('name', 'NotSet'))
# fields
fieldnames = bi.get('Field', {})
if isinstance(fieldnames, list):
for f in fieldnames:
signatures_ids.append(f.get('name', 'NotSet'))
elif isinstance(fieldnames, dict):
signatures_ids.append(fieldnames.get('name', 'NotSet'))
type = bi['type']
signature = '%s||%s||%s' % (bug_category, type,
'||'.join(signatures_ids))
# method
if signature not in bug_list:
bug_list.append(signature)
proj_array_count.incr(bug_category)
proj_array_count.incr('TOTAL_' + bug_category)
print proj_array_count.get_series()
results['%s||%s' % (p.group_id(),
p.artifact_id())] = proj_array_count.get_series()
save_to_file('bug_correlation_counters.json', json.dumps(results))