def main(): projects = load_projects_json() total_projects = len(projects) count = 0 bugless_count = 0 print 'Found %d Projects' % (total_projects, ) for p in projects: piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type' ]) doc_list = piter.documents_list() proj_array_count = ArrayCount() bug_list = [] count += 1 for d in doc_list: bug_instances = d.get('BugCollection', {}).get('BugInstance', []) if len(bug_instances) == 0: bugless_count += 1 break print '[%d:%d:%d] %s||%s: %d versions' % ( count, total_projects, bugless_count, p.group_id(), p.artifact_id(), len(doc_list)) print "bugless: %d, total: %d" % (bugless_count, total)
def main(): versions = [] dup_versions = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] version = d['JarMetadata']['version'] ga = '%s||%s||%s' % (group_id, artifact_id, version) if ga not in versions: versions.append(ga) else: dup_versions.incr(ga) print '[%d:%d:%d]: Processed %s' % (dup_versions.item_count(), len(versions), miter.count(), ga) print 'Total documents: %d, dups: %d, versions: %d' % (miter.total(), dup_versions.item_count(), len(versions)) save_to_file('duplicates.json', json.dumps(dup_versions.get_series()))
def main(): statistics = ArrayCount() for p in load_projects_json(): statistics.incr(p.version_count()) strio = StringIO.StringIO() for (k, v) in statistics.get_series().iteritems(): strio.write(str(k) + "," + str(v) + "\n") save_to_file('version_count.dat', strio.getvalue())
def main(): results = ArrayCount() miter = MongoDocumentIterator(fields=['JarMetadata.group_id', 'JarMetadata.artifact_id']) print 'Found %d Documents' % (miter.total(),) while miter.has_next(): d = miter.next() if d is not None: group_id = d['JarMetadata']['group_id'] artifact_id = d['JarMetadata']['artifact_id'] ga = '%s||%s' % (group_id, artifact_id) results.incr(ga) print 'Working %d of %d' % (miter.count(), miter.total(),) save_to_file('project_versions.json', json.dumps(results.get_series()))
def main(): fp = open('data/bug_correlation_counters_full.json', 'r') json_corr = json.load(fp) fp.close() totals = ArrayCount() for (k, v) in json_corr.iteritems(): if len(v) > 0: for (key, value) in v.iteritems(): totals.incr(key, delta=value) total = 0 for (k, v) in totals.get_series().iteritems(): if k.startswith('TOTAL_'): total += v print 'Total: %d' % (total,) for (k, v) in totals.get_series().iteritems(): if k.startswith('TOTAL_'): print '%s %.2f' % (k.replace('TOTAL_', '').title(), (float(v) / float(total))*100)
def main(): projects = load_vuln_projects_json() results = {} security_bugs = [ 'HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER' ] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects, ) for p in projects: piter = MongoProjectIterator( p.group_id(), p.artifact_id(), fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname', 'BugCollection.BugInstance.priority' ]) doc_list = piter.documents_list() documents = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: doc_results = {'JarMetadata': d['JarMetadata']} doc_array_count = ArrayCount() sec_instances = [] for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): print bi continue bug_category = bi.get('category', '') # method if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE': classnames = bi['Class'] classresults = [] if isinstance(classnames, list): for c in classnames: classresults.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): classresults.append( classnames.get('classname', 'NotSet')) sec_dict = { 'Category': bug_category, 'Type': bi.get('type', 'NotSet'), 'Priority': int(bi.get('priority', 0)), 'Class': classresults } sec_instances.append(sec_dict) # counters if bug_category == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: doc_array_count.incr('SECURITY_HIGH') else: doc_array_count.incr('SECURITY_LOW') else: doc_array_count.incr(bug_category) #doc_array_count.incr(bug_category) doc_results['Counters'] = doc_array_count.get_series() doc_results['SecurityBugs'] = sec_instances documents.append(doc_results) key = '%s||%s' % (p.group_id(), p.artifact_id()) results[key] = { 'group_id': p.group_id(), 'artifact_id': p.artifact_id(), 'version_count': len(doc_list), 'versions': documents } #print results save_to_file('project_counters.json', json.dumps(results))
def main(): projects = load_evolution_projects_json() results = {} security_bugs = [ 'HRS_REQUEST_PARAMETER_TO_COOKIE', 'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL', 'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE', 'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING', 'XSS_REQUEST_PARAMETER_TO_JSP_WRITER', 'XSS_REQUEST_PARAMETER_TO_SEND_ERROR', 'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER' ] total_projects = len(projects) count = 0 print 'Found %d Projects' % (total_projects, ) for p in projects: piter = MongoProjectIterator( p.group_id(), p.artifact_id(), fields=[ 'JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname', 'BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name' ]) doc_list = piter.documents_list() proj_array_count = ArrayCount() bug_list = [] count += 1 print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list)) for d in doc_list: for bi in d.get('BugCollection', {}).get('BugInstance', []): if not isinstance(bi, dict): #print 'Invalid BugInstance (%s)' % (bi,) continue bug_c = bi.get('category', '') if bug_c == 'SECURITY': bug_type = bi.get('type', None) if bug_type is None: print 'Invalid Type!' continue if bug_type in security_bugs: bug_category = 'SECURITY_HIGH' else: bug_category = 'SECURITY_LOW' else: bug_category = bug_c # create signature signatures_ids = [] classnames = bi['Class'] if isinstance(classnames, list): for c in classnames: signatures_ids.append(c.get('classname', 'NotSet')) elif isinstance(classnames, dict): signatures_ids.append(classnames.get( 'classname', 'NotSet')) # methods methodnames = bi.get('Method', {}) if isinstance(methodnames, list): for m in methodnames: signatures_ids.append(m.get('name', 'NotSet')) elif isinstance(methodnames, dict): signatures_ids.append(methodnames.get('name', 'NotSet')) # fields fieldnames = bi.get('Field', {}) if isinstance(fieldnames, list): for f in fieldnames: signatures_ids.append(f.get('name', 'NotSet')) elif isinstance(fieldnames, dict): signatures_ids.append(fieldnames.get('name', 'NotSet')) type = bi['type'] signature = '%s||%s||%s' % (bug_category, type, '||'.join(signatures_ids)) # method if signature not in bug_list: bug_list.append(signature) proj_array_count.incr(bug_category) proj_array_count.incr('TOTAL_' + bug_category) print proj_array_count.get_series() results['%s||%s' % (p.group_id(), p.artifact_id())] = proj_array_count.get_series() save_to_file('bug_correlation_counters.json', json.dumps(results))