def main(): versions = [] dup_versions = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] version = d['JarMetadata']['version'] ga = '%s||%s||%s' % (group_id, artifact_id, version) if ga not in versions: versions.append(ga) else: dup_versions.incr(ga) print '[%d:%d:%d]: Processed %s' % (dup_versions.item_count(), len(versions), miter.count(), ga) print 'Total documents: %d, dups: %d, versions: %d' % (miter.total(), dup_versions.item_count(), len(versions)) save_to_file('duplicates.json', json.dumps(dup_versions.get_series()))
def main(): statistics = ArrayCount() for p in load_projects_json(): statistics.incr(p.version_count()) strio = StringIO.StringIO() for (k, v) in statistics.get_series().iteritems(): strio.write(str(k) + "," + str(v) + "\n") save_to_file('version_count.dat', strio.getvalue())
def main(): results = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] ga = '%s||%s' % (group_id, artifact_id) results.incr(ga) print 'Working %d of %d' % (miter.count(), miter.total(),) save_to_file('project_versions.json', json.dumps(results.get_series()))
def main(): project_list = [] miter = MongoDocumentIterator( query={'BugCollection.BugInstance.category': 'SECURITY'}, fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d documents with SECURITY bugs' % (miter.total(), ) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (security)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) miter = MongoDocumentIterator( query={'BugCollection.BugInstance.category': 'MALICIOUS_CODE'}, fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type' ]) print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(), ) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (malicious_code)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) print "Total: %d Projects" % (len(project_list), ) save_to_file('vuln_projects.json', json.dumps(project_list))
def main(): project_list = [] miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'SECURITY'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d documents with SECURITY bugs' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (security)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) miter = MongoDocumentIterator(query={'BugCollection.BugInstance.category':'MALICIOUS_CODE'},fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version','BugCollection.BugInstance.category', 'BugCollection.BugInstance.type']) print 'Found %d documents with MALICIOUS_CODE bugs' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: print '%d of %d (malicious_code)' % (miter.count(), miter.total()) group_id = d.get('JarMetadata', {}).get('group_id', 'NotSet') artifact_id = d.get('JarMetadata', {}).get('artifact_id', 'NotSet') project_key = '%s||%s' % (group_id, artifact_id) if project_key not in project_list: project_list.append(project_key) print "Total: %d Projects" % (len(project_list),) save_to_file('vuln_projects.json', json.dumps(project_list))
def main(): projects = load_projects_json() valid_projects = [] total = len(projects) valid = 0 counter = 0 for p in projects: counter += 1 key = '%s||%s' % (p.group_id(), p.artifact_id()) piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.version_order'])\ piter.evolution_list() print '[%d:%d:%d] Checking ... %s' % (counter, valid, total, key), if piter.valid(): valid_projects.append(key) print ' ... Valid (%d versions)' % (len(piter.evolution_list())) valid += 1 else: print ' ... Invalid (%d versions)' % (len(piter.evolution_list())) print 'Total: %d, Valid: %d' % (total, valid) save_to_file('valid_projects.json', json.dumps(valid_projects))
def main(): projects = load_vuln_projects_json() results = {} security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects,) for p in projects: piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority']) doc_list = piter.documents_list() documents = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: doc_results = {'JarMetadata': d['JarMetadata']} doc_array_count = ArrayCount() sec_instances = [] for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): print bi continue bug_category = bi.get('category', '') # method if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE': classnames = bi['Class'] classresults = [] if isinstance(classnames, list): for c in classnames: classresults.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): classresults.append(classnames.get('classname', 'NotSet')) sec_dict = {'Category' : bug_category, 'Type' : bi.get('type', 'NotSet'), 'Priority' : int(bi.get('priority', 0)), 'Class' : classresults} sec_instances.append(sec_dict) # counters if bug_category == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: doc_array_count.incr('SECURITY_HIGH') else: doc_array_count.incr('SECURITY_LOW') else: doc_array_count.incr(bug_category) #doc_array_count.incr(bug_category) doc_results['Counters'] = doc_array_count.get_series() doc_results['SecurityBugs'] = sec_instances documents.append(doc_results) key = '%s||%s' % (p.group_id(), p.artifact_id()) results[key] = {'group_id' : p.group_id(), 'artifact_id' : p.artifact_id(), 'version_count' : len(doc_list), 'versions' : documents} #print results save_to_file('project_counters.json', json.dumps(results))
def main(): projects = load_vuln_projects_json() results = {} security_bugs = [ 'HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER' ] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects, ) for p in projects: piter = MongoProjectIterator( p.group_id(), p.artifact_id(), fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname', 'BugCollection.BugInstance.priority' ]) doc_list = piter.documents_list() documents = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: doc_results = {'JarMetadata': d['JarMetadata']} doc_array_count = ArrayCount() sec_instances = [] for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): print bi continue bug_category = bi.get('category', '') # method if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE': classnames = bi['Class'] classresults = [] if isinstance(classnames, list): for c in classnames: classresults.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): classresults.append( classnames.get('classname', 'NotSet')) sec_dict = { 'Category': bug_category, 'Type': bi.get('type', 'NotSet'), 'Priority': int(bi.get('priority', 0)), 'Class': classresults } sec_instances.append(sec_dict) # counters if bug_category == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: doc_array_count.incr('SECURITY_HIGH') else: doc_array_count.incr('SECURITY_LOW') else: doc_array_count.incr(bug_category) #doc_array_count.incr(bug_category) doc_results['Counters'] = doc_array_count.get_series() doc_results['SecurityBugs'] = sec_instances documents.append(doc_results) key = '%s||%s' % (p.group_id(), p.artifact_id()) results[key] = { 'group_id': p.group_id(), 'artifact_id': p.artifact_id(), 'version_count': len(doc_list), 'versions': documents } #print results save_to_file('project_counters.json', json.dumps(results))
def main(): projects = load_vuln_projects_json() results = {} security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'] sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow', 'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql', 'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all', 'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel', 'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms', 'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence', 'servicemix-components', 'sesame', 'sonar-application', 'sqltool', 'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta', 'ujo-orm', 'xmlui'} xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap', 'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app', 'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet', 'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator', 'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth', 'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp', 'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces', 'org.apache.felix.webconsole', 'org.apache.sling.openidauth', 'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console', 'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'} input_bugs = set() input_bugs |= sql_bugs input_bugs |= xss_bugs total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects,) for p in projects: piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority']) doc_list = piter.documents_list() documents = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: doc_results = {'JarMetadata': d['JarMetadata']} doc_array_count = ArrayCount() sec_instances = [] for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): print bi continue bug_category = bi.get('category', '') # method if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE': classnames = bi['Class'] classresults = [] if isinstance(classnames, list): for c in classnames: classresults.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): classresults.append(classnames.get('classname', 'NotSet')) sec_dict = {'Category' : bug_category, 'Type' : bi.get('type', 'NotSet'), 'Priority' : int(bi.get('priority', 0)), 'Class' : classresults} sec_instances.append(sec_dict) # counters if bug_category == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: if p.artifact_id() in input_bugs: doc_array_count.incr('INPUT_VALIDATION_BUGS') else: continue else: doc_array_count.incr('SECURITY_REST') else: doc_array_count.incr(bug_category) #doc_array_count.incr(bug_category) doc_results['Counters'] = doc_array_count.get_series() doc_results['SecurityBugs'] = sec_instances documents.append(doc_results) key = '%s||%s' % (p.group_id(), p.artifact_id()) results[key] = {'group_id' : p.group_id(), 'artifact_id' : p.artifact_id(), 'version_count' : len(doc_list), 'versions' : documents} #print results save_to_file('data/project_counters.json', json.dumps(results))
def main(): projects = load_projects_json() results = {} security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects,) for p in projects: piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name']) doc_list = piter.documents_list() proj_array_count = ArrayCount() bug_list = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): #print 'Invalid BugInstance (%s)' % (bi,) continue bug_c = bi.get('category', '') if bug_c == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: bug_category = 'SECURITY_HIGH' else: bug_category = 'SECURITY_LOW' else: bug_category = bug_c # create signature signatures_ids = [] classnames = bi['Class'] if isinstance(classnames, list): for c in classnames: signatures_ids.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): signatures_ids.append(classnames.get('classname', 'NotSet')) # methods methodnames = bi.get('Method', {}) if isinstance(methodnames, list): for m in methodnames: signatures_ids.append(m.get('name', 'NotSet')) elif isinstance(methodnames, dict): signatures_ids.append(methodnames.get('name', 'NotSet')) # fields fieldnames = bi.get('Field', {}) if isinstance(fieldnames, list): for f in fieldnames: signatures_ids.append(f.get('name', 'NotSet')) elif isinstance(fieldnames, dict): signatures_ids.append(fieldnames.get('name', 'NotSet')) type = bi['type'] signature = '%s||%s||%s' % (bug_category, type, '||'.join(signatures_ids)) # method if signature not in bug_list: bug_list.append(signature) proj_array_count.incr(bug_category) proj_array_count.incr('TOTAL_' + bug_category) print proj_array_count.get_series() results['%s||%s' % (p.group_id(), p.artifact_id())] = proj_array_count.get_series() save_to_file('bug_correlation_counters_full.json', json.dumps(results))
try: if d is not None: print 'Working %d of %d' % (miter.count(), miter.total(),) node_key = '%s||%s||%s' % (d['JarMetadata']['group_id'], d['JarMetadata']['artifact_id'], d['JarMetadata']['version']) deps = [] for dep in d.get('JarMetadata', {}).get('dependencies', {}): if isinstance(dep, dict): dep_group_id = dep.get('groupId', None) dep_artifact_id = dep.get('artifactId', None) dep_version = dep.get('version', None) if dep_group_id is None or dep_artifact_id is None: continue if dep_version is None: deps.append('%s||%s' % (dep_group_id, dep_artifact_id)) else: deps.append('%s||%s||%s' % (dep_group_id, dep_artifact_id, dep_version)) results[node_key] = {'dependencies ' : deps, 'timestamp' : d['JarMetadata'].get('jar_last_modification_date', 0), 'version_order' : d['JarMetadata']['version_order']} except Exception, e: print d save_to_file('project_graph.json', json.dumps(results)) if __name__ == "__main__": main()
def main(): projects = load_evolution_projects_json() results = OrderedDict() total_projects = len(projects) security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'] sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow', 'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql', 'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all', 'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel', 'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms', 'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence', 'servicemix-components', 'sesame', 'sonar-application', 'sqltool', 'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta', 'ujo-orm', 'xmlui'} xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap', 'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app', 'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet', 'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator', 'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth', 'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp', 'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces', 'org.apache.felix.webconsole', 'org.apache.sling.openidauth', 'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console', 'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'} input_bugs = set() input_bugs |= sql_bugs input_bugs |= xss_bugs count = 0 print 'Found %d Projects' % (total_projects,) for p in projects: piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name']) doc_list = piter.documents_list() count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: if d['JarMetadata']['version_order'] == 0: continue proj_array_count = ArrayCount() signatures = [] for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): #print 'Invalid BugInstance (%s)' % (bi,) continue bug_c = bi.get('category', '') if bug_c == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: if p.artifact_id() in input_bugs: bug_category = 'INPUT_VALIDATION_BUGS' else: continue else: bug_category = 'SECURITY_REST' else: bug_category = bug_c # create signature signatures_ids = [] classnames = bi['Class'] if isinstance(classnames, list): for c in classnames: signatures_ids.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): signatures_ids.append(classnames.get('classname', 'NotSet')) # methods methodnames = bi.get('Method', {}) if isinstance(methodnames, list): for m in methodnames: signatures_ids.append(m.get('name', 'NotSet')) elif isinstance(methodnames, dict): signatures_ids.append(methodnames.get('name', 'NotSet')) # fields fieldnames = bi.get('Field', {}) if isinstance(fieldnames, list): for f in fieldnames: signatures_ids.append(f.get('name', 'NotSet')) elif isinstance(fieldnames, dict): signatures_ids.append(fieldnames.get('name', 'NotSet')) bug_type = bi['type'] signature = '%s||%s||%s' % (bug_category, bug_type, '||'.join(signatures_ids)) signatures.append(signature) proj_array_count.incr('bug_category') results['%s||%s||%s' % (p.group_id(), p.artifact_id(), d['JarMetadata']['version'])] = {'Counters': proj_array_count.get_series(), 'Bugs': signatures, 'version_order': d['JarMetadata']['version_order']} save_to_file('data/bug_persistence.json', json.dumps(results))
def main(): projects = load_evolution_projects_json() results = {} security_bugs = [ 'HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER' ] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects, ) for p in projects: piter = MongoProjectIterator( p.group_id(), p.artifact_id(), fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname', 'BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name' ]) doc_list = piter.documents_list() proj_array_count = ArrayCount() bug_list = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): #print 'Invalid BugInstance (%s)' % (bi,) continue bug_c = bi.get('category', '') if bug_c == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: bug_category = 'SECURITY_HIGH' else: bug_category = 'SECURITY_LOW' else: bug_category = bug_c # create signature signatures_ids = [] classnames = bi['Class'] if isinstance(classnames, list): for c in classnames: signatures_ids.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): signatures_ids.append(classnames.get( 'classname', 'NotSet')) # methods methodnames = bi.get('Method', {}) if isinstance(methodnames, list): for m in methodnames: signatures_ids.append(m.get('name', 'NotSet')) elif isinstance(methodnames, dict): signatures_ids.append(methodnames.get('name', 'NotSet')) # fields fieldnames = bi.get('Field', {}) if isinstance(fieldnames, list): for f in fieldnames: signatures_ids.append(f.get('name', 'NotSet')) elif isinstance(fieldnames, dict): signatures_ids.append(fieldnames.get('name', 'NotSet')) type = bi['type'] signature = '%s||%s||%s' % (bug_category, type, '||'.join(signatures_ids)) # method if signature not in bug_list: bug_list.append(signature) proj_array_count.incr(bug_category) proj_array_count.incr('TOTAL_' + bug_category) print proj_array_count.get_series() results['%s||%s' % (p.group_id(), p.artifact_id())] = proj_array_count.get_series() save_to_file('bug_correlation_counters.json', json.dumps(results))
dep_artifact_id = dep.get('artifactId', None) dep_version = dep.get('version', None) if dep_group_id is None or dep_artifact_id is None: continue if dep_version is None: deps.append('%s||%s' % (dep_group_id, dep_artifact_id)) else: deps.append( '%s||%s||%s' % (dep_group_id, dep_artifact_id, dep_version)) results[node_key] = { 'dependencies ': deps, 'timestamp': d['JarMetadata'].get('jar_last_modification_date', 0), 'version_order': d['JarMetadata']['version_order'] } except Exception, e: print d save_to_file('project_graph.json', json.dumps(results)) if __name__ == "__main__": main()
def main(): projects = load_evolution_projects_json() results = {} security_bugs = [ "HRS_REQUEST_PARAMETER_TO_COOKIE", "HRS_REQUEST_PARAMETER_TO_HTTP_HEADER", "SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE", "SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING", "XSS_REQUEST_PARAMETER_TO_JSP_WRITER", "XSS_REQUEST_PARAMETER_TO_SEND_ERROR", "XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER", ] sql_bugs = { "activemq-all", "activemq", "activeobjects", "cas-workflow", "ebxmlms", "efaps-kernel", "fabric3-binding-ws", "geotk-metadata-sql", "jackrabbit-standalone", "james", "james-server-mailets", "jcaptcha-all", "jdatabaseimport", "jetty-webapp", "jonas-jms-manager", "joram", "kernel", "makumba", "MetaModel", "nunaliit2-adhocQueries", "openjms", "org.openl.rules.eclipse.ui.wizard", "sandesha2-persistence", "servicemix-components", "sesame", "sonar-application", "sqltool", "sqltool-j5", "squirrel-sql", "torque", "transactions-jta", "ujo-orm", "xmlui", } xss_bugs = { "activemq-all", "activemq-web", "makumba", "netcdf", "opendap", "org.talend.esb.job.console", "rdfbean-sparql", "tika-app", "tuscany-domain-manager", "tuscany-sca-all", "webmin", "WebProxyPortlet", "whiteboard", "activemq", "apacheds", "avro-tools", "css-validator", "dspace-jspui-api", "dspace-lni-core", "fabric3-binding-ws", "force-oauth", "groovysoap-all-jsr06", "jackrabbit-standalone", "jetty-webapp", "jftp", "makumba", "MessAdmin-Core", "myfaces", "myfaces-all", "ocpsoft-pretty-faces", "org.apache.felix.webconsole", "org.apache.sling.openidauth", "org.jbundle.util.webapp.redirect", "org.talend.esb.job.console", "pustefix-webservices-jaxws", "sonar-application", "vt-ldap", } input_bugs = set() input_bugs |= sql_bugs input_bugs |= xss_bugs total_projects = len(projects) count = 0 print "Found %d Projects" % (total_projects,) for p in projects: piter = MongoProjectIterator( p.group_id(), p.artifact_id(), fields=[ "JarMetadata.group_id", "JarMetadata.artifact_id", "JarMetadata.version", "JarMetadata.version_order", "BugCollection.BugInstance.category", "BugCollection.BugInstance.type", "BugCollection.BugInstance.Class.classname", "BugCollection.BugInstance.Method.name", "BugCollection.BugInstance.Field.name", ], ) doc_list = piter.documents_list() proj_array_count = ArrayCount() bug_list = [] count += 1 print "[%d:%d] %s||%s: %d versions" % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: for bi in d.get("BugCollection", {}).get("BugInstance", []): if not isinstance(bi, dict): # print 'Invalid BugInstance (%s)' % (bi,) continue bug_c = bi.get("category", "") if bug_c == "SECURITY": bug_type = bi.get("type", None) if bug_type is None: print "Invalid Type!" continue if bug_type in security_bugs: if p.artifact_id() in input_bugs: bug_category = "INPUT_VALIDATION_BUGS" else: continue else: bug_category = "SECURITY_REST" else: bug_category = bug_c # create signature signatures_ids = [] classnames = bi["Class"] if isinstance(classnames, list): for c in classnames: signatures_ids.append(c.get("classname", "NotSet")) elif isinstance(classnames, dict): signatures_ids.append(classnames.get("classname", "NotSet")) # methods methodnames = bi.get("Method", {}) if isinstance(methodnames, list): for m in methodnames: signatures_ids.append(m.get("name", "NotSet")) elif isinstance(methodnames, dict): signatures_ids.append(methodnames.get("name", "NotSet")) # fields fieldnames = bi.get("Field", {}) if isinstance(fieldnames, list): for f in fieldnames: signatures_ids.append(f.get("name", "NotSet")) elif isinstance(fieldnames, dict): signatures_ids.append(fieldnames.get("name", "NotSet")) type = bi["type"] signature = "%s||%s||%s" % (bug_category, type, "||".join(signatures_ids)) # method if signature not in bug_list: bug_list.append(signature) proj_array_count.incr(bug_category) proj_array_count.incr("TOTAL_" + bug_category) print proj_array_count.get_series() results["%s||%s" % (p.group_id(), p.artifact_id())] = proj_array_count.get_series() save_to_file("data/bug_correlation_counters.json", json.dumps(results))