def main():
statistics = ArrayCount()
for p in load_projects_json():
statistics.incr(p.version_count())
strio = StringIO.StringIO()
for (k, v) in statistics.get_series().iteritems():
strio.write(str(k) + "," + str(v) + "\n")
save_to_file('version_count.dat', strio.getvalue())
def main():
projects = load_projects_json()
total_projects = len(projects)
count = 0
bugless_count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id',
'JarMetadata.artifact_id',
'JarMetadata.version',
'JarMetadata.version_order',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type'
])
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
for d in doc_list:
bug_instances = d.get('BugCollection', {}).get('BugInstance', [])
if len(bug_instances) == 0:
bugless_count += 1
break
print '[%d:%d:%d] %s||%s: %d versions' % (
count, total_projects, bugless_count, p.group_id(),
p.artifact_id(), len(doc_list))
print "bugless: %d, total: %d" % (bugless_count, total)
def main():
results = ArrayCount()
miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id'])
print 'Found %d Documents' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
group_id = d['JarMetadata']['group_id']
artifact_id = d['JarMetadata']['artifact_id']
ga = '%s||%s' % (group_id, artifact_id)
results.incr(ga)
print 'Working %d of %d' % (miter.count(), miter.total(),)
save_to_file('project_versions.json', json.dumps(results.get_series()))
def main():
versions = []
dup_versions = ArrayCount()
miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version'])
print 'Found %d Documents' % (miter.total(),)
while miter.has_next():
d = miter.next()
if d is not None:
group_id = d['JarMetadata']['group_id']
artifact_id = d['JarMetadata']['artifact_id']
version = d['JarMetadata']['version']
ga = '%s||%s||%s' % (group_id, artifact_id, version)
if ga not in versions:
versions.append(ga)
else:
dup_versions.incr(ga)
print '[%d:%d:%d]: Processed %s' % (dup_versions.item_count(), len(versions), miter.count(), ga)
print 'Total documents: %d, dups: %d, versions: %d' % (miter.total(), dup_versions.item_count(), len(versions))
save_to_file('duplicates.json', json.dumps(dup_versions.get_series()))
def main():
fp = open("data/bug_correlation_counters_full.json", "r")
json_corr = json.load(fp)
fp.close()
totals = ArrayCount()
for (k, v) in json_corr.iteritems():
if len(v) > 0:
for (key, value) in v.iteritems():
totals.incr(key, delta=value)
total = 0
for (k, v) in totals.get_series().iteritems():
if k.startswith("TOTAL_"):
total += v
print "Total: %d" % (total,)
for (k, v) in totals.get_series().iteritems():
if k.startswith("TOTAL_"):
print "%s %.2f" % (k.replace("TOTAL_", "").title(), (float(v) / float(total)) * 100)
def main():
fp = open('data/bug_correlation_counters_full.json', 'r')
json_corr = json.load(fp)
fp.close()
totals = ArrayCount()
for (k, v) in json_corr.iteritems():
if len(v) > 0:
for (key, value) in v.iteritems():
totals.incr(key, delta=value)
total = 0
for (k, v) in totals.get_series().iteritems():
if k.startswith('TOTAL_'):
total += v
print 'Total: %d' % (total,)
for (k, v) in totals.get_series().iteritems():
if k.startswith('TOTAL_'):
print '%s %.2f' % (k.replace('TOTAL_', '').title(), (float(v) / float(total))*100)
def main():
projects = load_vuln_projects_json()
results = {}
security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects,)
for p in projects:
piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority'])
doc_list = piter.documents_list()
documents = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))
for d in doc_list:
doc_results = {'JarMetadata': d['JarMetadata']}
doc_array_count = ArrayCount()
sec_instances = []
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
print bi
continue
bug_category = bi.get('category', '')
# method
if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
classnames = bi['Class']
classresults = []
if isinstance(classnames, list):
for c in classnames:
classresults.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
classresults.append(classnames.get('classname', 'NotSet'))
sec_dict = {'Category' : bug_category,
'Type' : bi.get('type', 'NotSet'),
'Priority' : int(bi.get('priority', 0)),
'Class' : classresults}
sec_instances.append(sec_dict)
# counters
if bug_category == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
doc_array_count.incr('SECURITY_HIGH')
else:
doc_array_count.incr('SECURITY_LOW')
else:
doc_array_count.incr(bug_category)
#doc_array_count.incr(bug_category)
doc_results['Counters'] = doc_array_count.get_series()
doc_results['SecurityBugs'] = sec_instances
documents.append(doc_results)
key = '%s||%s' % (p.group_id(), p.artifact_id())
results[key] = {'group_id' : p.group_id(),
'artifact_id' : p.artifact_id(),
'version_count' : len(doc_list),
'versions' : documents}
#print results
save_to_file('project_counters.json', json.dumps(results))
def main():
projects = load_vuln_projects_json()
results = {}
security_bugs = [
'HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
]
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(
p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id', 'JarMetadata.artifact_id',
'JarMetadata.version', 'JarMetadata.jar_size',
'JarMetadata.version_order',
'JarMetadata.jar_last_modification_date',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type',
'BugCollection.BugInstance.Class.classname',
'BugCollection.BugInstance.priority'
])
doc_list = piter.documents_list()
documents = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
p.group_id(), p.artifact_id(),
len(doc_list))
for d in doc_list:
doc_results = {'JarMetadata': d['JarMetadata']}
doc_array_count = ArrayCount()
sec_instances = []
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
print bi
continue
bug_category = bi.get('category', '')
# method
if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
classnames = bi['Class']
classresults = []
if isinstance(classnames, list):
for c in classnames:
classresults.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
classresults.append(
classnames.get('classname', 'NotSet'))
sec_dict = {
'Category': bug_category,
'Type': bi.get('type', 'NotSet'),
'Priority': int(bi.get('priority', 0)),
'Class': classresults
}
sec_instances.append(sec_dict)
# counters
if bug_category == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
doc_array_count.incr('SECURITY_HIGH')
else:
doc_array_count.incr('SECURITY_LOW')
else:
doc_array_count.incr(bug_category)
#doc_array_count.incr(bug_category)
doc_results['Counters'] = doc_array_count.get_series()
doc_results['SecurityBugs'] = sec_instances
documents.append(doc_results)
key = '%s||%s' % (p.group_id(), p.artifact_id())
results[key] = {
'group_id': p.group_id(),
'artifact_id': p.artifact_id(),
'version_count': len(doc_list),
'versions': documents
}
#print results
save_to_file('project_counters.json', json.dumps(results))
def main():
projects = load_vuln_projects_json()
results = {}
security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']
sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow',
'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql',
'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all',
'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel',
'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms',
'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence',
'servicemix-components', 'sesame', 'sonar-application', 'sqltool',
'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta',
'ujo-orm', 'xmlui'}
xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap',
'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app',
'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet',
'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator',
'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth',
'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp',
'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces',
'org.apache.felix.webconsole', 'org.apache.sling.openidauth',
'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console',
'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'}
input_bugs = set()
input_bugs |= sql_bugs
input_bugs |= xss_bugs
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects,)
for p in projects:
piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority'])
doc_list = piter.documents_list()
documents = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))
for d in doc_list:
doc_results = {'JarMetadata': d['JarMetadata']}
doc_array_count = ArrayCount()
sec_instances = []
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
print bi
continue
bug_category = bi.get('category', '')
# method
if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
classnames = bi['Class']
classresults = []
if isinstance(classnames, list):
for c in classnames:
classresults.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
classresults.append(classnames.get('classname', 'NotSet'))
sec_dict = {'Category' : bug_category,
'Type' : bi.get('type', 'NotSet'),
'Priority' : int(bi.get('priority', 0)),
'Class' : classresults}
sec_instances.append(sec_dict)
# counters
if bug_category == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
if p.artifact_id() in input_bugs:
doc_array_count.incr('INPUT_VALIDATION_BUGS')
else:
continue
else:
doc_array_count.incr('SECURITY_REST')
else:
doc_array_count.incr(bug_category)
#doc_array_count.incr(bug_category)
doc_results['Counters'] = doc_array_count.get_series()
doc_results['SecurityBugs'] = sec_instances
documents.append(doc_results)
key = '%s||%s' % (p.group_id(), p.artifact_id())
results[key] = {'group_id' : p.group_id(),
'artifact_id' : p.artifact_id(),
'version_count' : len(doc_list),
'versions' : documents}
#print results
save_to_file('data/project_counters.json', json.dumps(results))
def main():
projects = load_projects_json()
results = {}
security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects,)
for p in projects:
piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name'])
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))
for d in doc_list:
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
#print 'Invalid BugInstance (%s)' % (bi,)
continue
bug_c = bi.get('category', '')
if bug_c == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
bug_category = 'SECURITY_HIGH'
else:
bug_category = 'SECURITY_LOW'
else:
bug_category = bug_c
# create signature
signatures_ids = []
classnames = bi['Class']
if isinstance(classnames, list):
for c in classnames:
signatures_ids.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
signatures_ids.append(classnames.get('classname', 'NotSet'))
# methods
methodnames = bi.get('Method', {})
if isinstance(methodnames, list):
for m in methodnames:
signatures_ids.append(m.get('name', 'NotSet'))
elif isinstance(methodnames, dict):
signatures_ids.append(methodnames.get('name', 'NotSet'))
# fields
fieldnames = bi.get('Field', {})
if isinstance(fieldnames, list):
for f in fieldnames:
signatures_ids.append(f.get('name', 'NotSet'))
elif isinstance(fieldnames, dict):
signatures_ids.append(fieldnames.get('name', 'NotSet'))
type = bi['type']
signature = '%s||%s||%s' % (bug_category, type, '||'.join(signatures_ids))
# method
if signature not in bug_list:
bug_list.append(signature)
proj_array_count.incr(bug_category)
proj_array_count.incr('TOTAL_' + bug_category)
print proj_array_count.get_series()
results['%s||%s' % (p.group_id(), p.artifact_id())] = proj_array_count.get_series()
save_to_file('bug_correlation_counters_full.json', json.dumps(results))
def main():
projects = load_evolution_projects_json()
results = OrderedDict()
total_projects = len(projects)
security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']
sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow',
'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql',
'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all',
'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel',
'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms',
'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence',
'servicemix-components', 'sesame', 'sonar-application', 'sqltool',
'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta',
'ujo-orm', 'xmlui'}
xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap',
'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app',
'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet',
'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator',
'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth',
'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp',
'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces',
'org.apache.felix.webconsole', 'org.apache.sling.openidauth',
'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console',
'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'}
input_bugs = set()
input_bugs |= sql_bugs
input_bugs |= xss_bugs
count = 0
print 'Found %d Projects' % (total_projects,)
for p in projects:
piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name'])
doc_list = piter.documents_list()
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))
for d in doc_list:
if d['JarMetadata']['version_order'] == 0:
continue
proj_array_count = ArrayCount()
signatures = []
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
#print 'Invalid BugInstance (%s)' % (bi,)
continue
bug_c = bi.get('category', '')
if bug_c == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
if p.artifact_id() in input_bugs:
bug_category = 'INPUT_VALIDATION_BUGS'
else:
continue
else:
bug_category = 'SECURITY_REST'
else:
bug_category = bug_c
# create signature
signatures_ids = []
classnames = bi['Class']
if isinstance(classnames, list):
for c in classnames:
signatures_ids.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
signatures_ids.append(classnames.get('classname', 'NotSet'))
# methods
methodnames = bi.get('Method', {})
if isinstance(methodnames, list):
for m in methodnames:
signatures_ids.append(m.get('name', 'NotSet'))
elif isinstance(methodnames, dict):
signatures_ids.append(methodnames.get('name', 'NotSet'))
# fields
fieldnames = bi.get('Field', {})
if isinstance(fieldnames, list):
for f in fieldnames:
signatures_ids.append(f.get('name', 'NotSet'))
elif isinstance(fieldnames, dict):
signatures_ids.append(fieldnames.get('name', 'NotSet'))
bug_type = bi['type']
signature = '%s||%s||%s' % (bug_category, bug_type, '||'.join(signatures_ids))
signatures.append(signature)
proj_array_count.incr('bug_category')
results['%s||%s||%s' % (p.group_id(), p.artifact_id(), d['JarMetadata']['version'])] = {'Counters': proj_array_count.get_series(), 'Bugs': signatures, 'version_order': d['JarMetadata']['version_order']}
save_to_file('data/bug_persistence.json', json.dumps(results))
def main():
projects = load_evolution_projects_json()
results = {}
security_bugs = [
'HRS_REQUEST_PARAMETER_TO_COOKIE',
'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
]
total_projects = len(projects)
count = 0
print 'Found %d Projects' % (total_projects, )
for p in projects:
piter = MongoProjectIterator(
p.group_id(),
p.artifact_id(),
fields=[
'JarMetadata.group_id', 'JarMetadata.artifact_id',
'JarMetadata.version', 'JarMetadata.version_order',
'BugCollection.BugInstance.category',
'BugCollection.BugInstance.type',
'BugCollection.BugInstance.Class.classname',
'BugCollection.BugInstance.Method.name',
'BugCollection.BugInstance.Field.name'
])
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
p.group_id(), p.artifact_id(),
len(doc_list))
for d in doc_list:
for bi in d.get('BugCollection', {}).get('BugInstance', []):
if not isinstance(bi, dict):
#print 'Invalid BugInstance (%s)' % (bi,)
continue
bug_c = bi.get('category', '')
if bug_c == 'SECURITY':
bug_type = bi.get('type', None)
if bug_type is None:
print 'Invalid Type!'
continue
if bug_type in security_bugs:
bug_category = 'SECURITY_HIGH'
else:
bug_category = 'SECURITY_LOW'
else:
bug_category = bug_c
# create signature
signatures_ids = []
classnames = bi['Class']
if isinstance(classnames, list):
for c in classnames:
signatures_ids.append(c.get('classname', 'NotSet'))
elif isinstance(classnames, dict):
signatures_ids.append(classnames.get(
'classname', 'NotSet'))
# methods
methodnames = bi.get('Method', {})
if isinstance(methodnames, list):
for m in methodnames:
signatures_ids.append(m.get('name', 'NotSet'))
elif isinstance(methodnames, dict):
signatures_ids.append(methodnames.get('name', 'NotSet'))
# fields
fieldnames = bi.get('Field', {})
if isinstance(fieldnames, list):
for f in fieldnames:
signatures_ids.append(f.get('name', 'NotSet'))
elif isinstance(fieldnames, dict):
signatures_ids.append(fieldnames.get('name', 'NotSet'))
type = bi['type']
signature = '%s||%s||%s' % (bug_category, type,
'||'.join(signatures_ids))
# method
if signature not in bug_list:
bug_list.append(signature)
proj_array_count.incr(bug_category)
proj_array_count.incr('TOTAL_' + bug_category)
print proj_array_count.get_series()
results['%s||%s' % (p.group_id(),
p.artifact_id())] = proj_array_count.get_series()
save_to_file('bug_correlation_counters.json', json.dumps(results))
def main():
projects = load_evolution_projects_json()
results = {}
security_bugs = [
"HRS_REQUEST_PARAMETER_TO_COOKIE",
"HRS_REQUEST_PARAMETER_TO_HTTP_HEADER",
"SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE",
"SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING",
"XSS_REQUEST_PARAMETER_TO_JSP_WRITER",
"XSS_REQUEST_PARAMETER_TO_SEND_ERROR",
"XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER",
]
sql_bugs = {
"activemq-all",
"activemq",
"activeobjects",
"cas-workflow",
"ebxmlms",
"efaps-kernel",
"fabric3-binding-ws",
"geotk-metadata-sql",
"jackrabbit-standalone",
"james",
"james-server-mailets",
"jcaptcha-all",
"jdatabaseimport",
"jetty-webapp",
"jonas-jms-manager",
"joram",
"kernel",
"makumba",
"MetaModel",
"nunaliit2-adhocQueries",
"openjms",
"org.openl.rules.eclipse.ui.wizard",
"sandesha2-persistence",
"servicemix-components",
"sesame",
"sonar-application",
"sqltool",
"sqltool-j5",
"squirrel-sql",
"torque",
"transactions-jta",
"ujo-orm",
"xmlui",
}
xss_bugs = {
"activemq-all",
"activemq-web",
"makumba",
"netcdf",
"opendap",
"org.talend.esb.job.console",
"rdfbean-sparql",
"tika-app",
"tuscany-domain-manager",
"tuscany-sca-all",
"webmin",
"WebProxyPortlet",
"whiteboard",
"activemq",
"apacheds",
"avro-tools",
"css-validator",
"dspace-jspui-api",
"dspace-lni-core",
"fabric3-binding-ws",
"force-oauth",
"groovysoap-all-jsr06",
"jackrabbit-standalone",
"jetty-webapp",
"jftp",
"makumba",
"MessAdmin-Core",
"myfaces",
"myfaces-all",
"ocpsoft-pretty-faces",
"org.apache.felix.webconsole",
"org.apache.sling.openidauth",
"org.jbundle.util.webapp.redirect",
"org.talend.esb.job.console",
"pustefix-webservices-jaxws",
"sonar-application",
"vt-ldap",
}
input_bugs = set()
input_bugs |= sql_bugs
input_bugs |= xss_bugs
total_projects = len(projects)
count = 0
print "Found %d Projects" % (total_projects,)
for p in projects:
piter = MongoProjectIterator(
p.group_id(),
p.artifact_id(),
fields=[
"JarMetadata.group_id",
"JarMetadata.artifact_id",
"JarMetadata.version",
"JarMetadata.version_order",
"BugCollection.BugInstance.category",
"BugCollection.BugInstance.type",
"BugCollection.BugInstance.Class.classname",
"BugCollection.BugInstance.Method.name",
"BugCollection.BugInstance.Field.name",
],
)
doc_list = piter.documents_list()
proj_array_count = ArrayCount()
bug_list = []
count += 1
print "[%d:%d] %s||%s: %d versions" % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))
for d in doc_list:
for bi in d.get("BugCollection", {}).get("BugInstance", []):
if not isinstance(bi, dict):
# print 'Invalid BugInstance (%s)' % (bi,)
continue
bug_c = bi.get("category", "")
if bug_c == "SECURITY":
bug_type = bi.get("type", None)
if bug_type is None:
print "Invalid Type!"
continue
if bug_type in security_bugs:
if p.artifact_id() in input_bugs:
bug_category = "INPUT_VALIDATION_BUGS"
else:
continue
else:
bug_category = "SECURITY_REST"
else:
bug_category = bug_c
# create signature
signatures_ids = []
classnames = bi["Class"]
if isinstance(classnames, list):
for c in classnames:
signatures_ids.append(c.get("classname", "NotSet"))
elif isinstance(classnames, dict):
signatures_ids.append(classnames.get("classname", "NotSet"))
# methods
methodnames = bi.get("Method", {})
if isinstance(methodnames, list):
for m in methodnames:
signatures_ids.append(m.get("name", "NotSet"))
elif isinstance(methodnames, dict):
signatures_ids.append(methodnames.get("name", "NotSet"))
# fields
fieldnames = bi.get("Field", {})
if isinstance(fieldnames, list):
for f in fieldnames:
signatures_ids.append(f.get("name", "NotSet"))
elif isinstance(fieldnames, dict):
signatures_ids.append(fieldnames.get("name", "NotSet"))
type = bi["type"]
signature = "%s||%s||%s" % (bug_category, type, "||".join(signatures_ids))
# method
if signature not in bug_list:
bug_list.append(signature)
proj_array_count.incr(bug_category)
proj_array_count.incr("TOTAL_" + bug_category)
print proj_array_count.get_series()
results["%s||%s" % (p.group_id(), p.artifact_id())] = proj_array_count.get_series()
save_to_file("data/bug_correlation_counters.json", json.dumps(results))