Beispiel #1
0
def main():
    projects = load_projects_json()
    total_projects = len(projects)
    count = 0
    bugless_count = 0

    print 'Found %d Projects' % (total_projects, )

    for p in projects:
        piter = MongoProjectIterator(p.group_id(),
                                     p.artifact_id(),
                                     fields=[
                                         'JarMetadata.group_id',
                                         'JarMetadata.artifact_id',
                                         'JarMetadata.version',
                                         'JarMetadata.version_order',
                                         'BugCollection.BugInstance.category',
                                         'BugCollection.BugInstance.type'
                                     ])
        doc_list = piter.documents_list()
        proj_array_count = ArrayCount()
        bug_list = []
        count += 1

        for d in doc_list:
            bug_instances = d.get('BugCollection', {}).get('BugInstance', [])
            if len(bug_instances) == 0:
                bugless_count += 1
                break

        print '[%d:%d:%d] %s||%s: %d versions' % (
            count, total_projects, bugless_count, p.group_id(),
            p.artifact_id(), len(doc_list))

    print "bugless: %d, total: %d" % (bugless_count, total)
def main():
    projects = load_projects_json()
    total_projects = len(projects)
    count = 0
    bugless_count = 0

    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type'])
        doc_list = piter.documents_list()
        proj_array_count = ArrayCount()
        bug_list = []
        count += 1

        for d in doc_list:
        	bug_instances = d.get('BugCollection', {}).get('BugInstance', [])
        	if len(bug_instances) == 0:
        		bugless_count += 1
        		break

        print '[%d:%d:%d] %s||%s: %d versions' % (count, total_projects, bugless_count, p.group_id(), p.artifact_id(), len(doc_list))

    print "bugless: %d, total: %d" % (bugless_count, total)
def main():
    projects = load_vuln_projects_json()
    results = {}
    security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
                     'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
                     'PT_ABSOLUTE_PATH_TRAVERSAL',
                     'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
                     'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
                     'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
                     'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
                     'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']

    total_projects = len(projects)
    count = 0
    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority'])
        doc_list = piter.documents_list()
        documents = []
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))

        for d in doc_list:
            doc_results = {'JarMetadata': d['JarMetadata']}
            doc_array_count = ArrayCount()
            sec_instances = []

            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    print bi
                    continue

                bug_category = bi.get('category', '')

                # method
                if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
                    classnames = bi['Class']
                    classresults = []

                    if isinstance(classnames, list):
                        for c in classnames:
                            classresults.append(c.get('classname', 'NotSet'))
                    elif isinstance(classnames, dict):
                        classresults.append(classnames.get('classname', 'NotSet'))

                    sec_dict = {'Category' : bug_category,
                                'Type' : bi.get('type', 'NotSet'),
                                'Priority' : int(bi.get('priority', 0)),
                                'Class' : classresults}
                    sec_instances.append(sec_dict)

                # counters
                if bug_category == 'SECURITY':
                    bug_type = bi.get('type', None)
                    
                    if bug_type is None:
                        print 'Invalid Type!'
                        continue
                        
                    if bug_type in security_bugs:
                        doc_array_count.incr('SECURITY_HIGH')
                    else:
                        doc_array_count.incr('SECURITY_LOW')
                else:
                    doc_array_count.incr(bug_category)
                #doc_array_count.incr(bug_category)

            doc_results['Counters'] = doc_array_count.get_series()
            doc_results['SecurityBugs'] = sec_instances
            documents.append(doc_results)

        key = '%s||%s' % (p.group_id(), p.artifact_id())
        results[key] = {'group_id' : p.group_id(),
                        'artifact_id' : p.artifact_id(),
                        'version_count' : len(doc_list),
                        'versions' : documents}
        #print results

    save_to_file('project_counters.json', json.dumps(results))
def main():
    projects = load_vuln_projects_json()
    results = {}
    security_bugs = [
        'HRS_REQUEST_PARAMETER_TO_COOKIE',
        'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
        'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
        'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
        'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
        'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
        'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
    ]

    total_projects = len(projects)
    count = 0
    print 'Found %d Projects' % (total_projects, )

    for p in projects:
        piter = MongoProjectIterator(
            p.group_id(),
            p.artifact_id(),
            fields=[
                'JarMetadata.group_id', 'JarMetadata.artifact_id',
                'JarMetadata.version', 'JarMetadata.jar_size',
                'JarMetadata.version_order',
                'JarMetadata.jar_last_modification_date',
                'BugCollection.BugInstance.category',
                'BugCollection.BugInstance.type',
                'BugCollection.BugInstance.Class.classname',
                'BugCollection.BugInstance.priority'
            ])
        doc_list = piter.documents_list()
        documents = []
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
                                               p.group_id(), p.artifact_id(),
                                               len(doc_list))

        for d in doc_list:
            doc_results = {'JarMetadata': d['JarMetadata']}
            doc_array_count = ArrayCount()
            sec_instances = []

            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    print bi
                    continue

                bug_category = bi.get('category', '')

                # method
                if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
                    classnames = bi['Class']
                    classresults = []

                    if isinstance(classnames, list):
                        for c in classnames:
                            classresults.append(c.get('classname', 'NotSet'))
                    elif isinstance(classnames, dict):
                        classresults.append(
                            classnames.get('classname', 'NotSet'))

                    sec_dict = {
                        'Category': bug_category,
                        'Type': bi.get('type', 'NotSet'),
                        'Priority': int(bi.get('priority', 0)),
                        'Class': classresults
                    }
                    sec_instances.append(sec_dict)

                # counters
                if bug_category == 'SECURITY':
                    bug_type = bi.get('type', None)

                    if bug_type is None:
                        print 'Invalid Type!'
                        continue

                    if bug_type in security_bugs:
                        doc_array_count.incr('SECURITY_HIGH')
                    else:
                        doc_array_count.incr('SECURITY_LOW')
                else:
                    doc_array_count.incr(bug_category)
                #doc_array_count.incr(bug_category)

            doc_results['Counters'] = doc_array_count.get_series()
            doc_results['SecurityBugs'] = sec_instances
            documents.append(doc_results)

        key = '%s||%s' % (p.group_id(), p.artifact_id())
        results[key] = {
            'group_id': p.group_id(),
            'artifact_id': p.artifact_id(),
            'version_count': len(doc_list),
            'versions': documents
        }
        #print results

    save_to_file('project_counters.json', json.dumps(results))
def main():
    projects = load_vuln_projects_json()
    results = {}
    security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
                     'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
                     'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
                     'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
                     'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
                     'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
                     'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']

    sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow',
                'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql',
                'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all',
                'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel',
                'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms',
                'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence',
                'servicemix-components', 'sesame', 'sonar-application', 'sqltool',
                'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta',
                'ujo-orm', 'xmlui'}

    xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap',
                'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app',
                'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet',
                'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator',
                'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth',
                'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp',
                'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces',
                'org.apache.felix.webconsole', 'org.apache.sling.openidauth',
                'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console',
                'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'}

    input_bugs = set()
    input_bugs |= sql_bugs
    input_bugs |= xss_bugs

    total_projects = len(projects)
    count = 0
    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.jar_size', 'JarMetadata.version_order', 'JarMetadata.jar_last_modification_date', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.priority'])
        doc_list = piter.documents_list()
        documents = []
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))

        for d in doc_list:
            doc_results = {'JarMetadata': d['JarMetadata']}
            doc_array_count = ArrayCount()
            sec_instances = []

            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    print bi
                    continue

                bug_category = bi.get('category', '')

                # method
                if bug_category == 'SECURITY' or bug_category == 'MALICIOUS_CODE':
                    classnames = bi['Class']
                    classresults = []

                    if isinstance(classnames, list):
                        for c in classnames:
                            classresults.append(c.get('classname', 'NotSet'))
                    elif isinstance(classnames, dict):
                        classresults.append(classnames.get('classname', 'NotSet'))

                    sec_dict = {'Category' : bug_category,
                                'Type' : bi.get('type', 'NotSet'),
                                'Priority' : int(bi.get('priority', 0)),
                                'Class' : classresults}
                    sec_instances.append(sec_dict)

                # counters
                if bug_category == 'SECURITY':
                    bug_type = bi.get('type', None)
                    
                    if bug_type is None:
                        print 'Invalid Type!'
                        continue
                        
                    if bug_type in security_bugs:
                        if p.artifact_id() in input_bugs:
                            doc_array_count.incr('INPUT_VALIDATION_BUGS')
                        else:
                            continue
                    else:
                        doc_array_count.incr('SECURITY_REST')
                else:
                    doc_array_count.incr(bug_category)
                #doc_array_count.incr(bug_category)

            doc_results['Counters'] = doc_array_count.get_series()
            doc_results['SecurityBugs'] = sec_instances
            documents.append(doc_results)

        key = '%s||%s' % (p.group_id(), p.artifact_id())
        results[key] = {'group_id' : p.group_id(),
                        'artifact_id' : p.artifact_id(),
                        'version_count' : len(doc_list),
                        'versions' : documents}
        #print results

    save_to_file('data/project_counters.json', json.dumps(results))
def main():
    projects = load_projects_json()
    results = {}
    security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
                     'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
                     'PT_ABSOLUTE_PATH_TRAVERSAL',
                     'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
                     'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
                     'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
                     'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
                     'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']
    total_projects = len(projects)
    count = 0

    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name'])
        doc_list = piter.documents_list()
        proj_array_count = ArrayCount()
        bug_list = []
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))

        for d in doc_list:
            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    #print 'Invalid BugInstance (%s)' % (bi,)
                    continue

                bug_c = bi.get('category', '')
                if bug_c == 'SECURITY':
                    bug_type = bi.get('type', None)
                    
                    if bug_type is None:
                        print 'Invalid Type!'
                        continue
                        
                    if bug_type in security_bugs:
                        bug_category = 'SECURITY_HIGH'
                    else:
                        bug_category = 'SECURITY_LOW'
                else:
                    bug_category = bug_c
                
                # create signature
                signatures_ids = []
                classnames = bi['Class']

                if isinstance(classnames, list):
                    for c in classnames:
                        signatures_ids.append(c.get('classname', 'NotSet'))
                elif isinstance(classnames, dict):
                    signatures_ids.append(classnames.get('classname', 'NotSet'))

                # methods
                methodnames = bi.get('Method', {})

                if isinstance(methodnames, list):
                    for m in methodnames:
                        signatures_ids.append(m.get('name', 'NotSet'))
                elif isinstance(methodnames, dict):
                    signatures_ids.append(methodnames.get('name', 'NotSet'))

                # fields
                fieldnames = bi.get('Field', {})
                if isinstance(fieldnames, list):
                    for f in fieldnames:
                        signatures_ids.append(f.get('name', 'NotSet'))
                elif isinstance(fieldnames, dict):
                    signatures_ids.append(fieldnames.get('name', 'NotSet'))

                type = bi['type']
                signature = '%s||%s||%s' % (bug_category, type, '||'.join(signatures_ids))

                # method
                if signature not in bug_list:
                    bug_list.append(signature)
                    proj_array_count.incr(bug_category)
                
                proj_array_count.incr('TOTAL_' + bug_category)

        print proj_array_count.get_series()
        results['%s||%s' % (p.group_id(), p.artifact_id())] = proj_array_count.get_series()

    save_to_file('bug_correlation_counters_full.json', json.dumps(results))
def main():
    projects = load_evolution_projects_json()
    security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
                     'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
                     'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
                     'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
                     'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
                     'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
                     'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']

    sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow',
                'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws',
                'geotk-metadata-sql',
                'jackrabbit-standalone', 'james', 'james-server-mailets',
                'jcaptcha-all',
                'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager',
                'joram', 'kernel',
                'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms',
                'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence',
                'servicemix-components', 'sesame', 'sonar-application',
                'sqltool',
                'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta',
                'ujo-orm', 'xmlui'}

    xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap',
                'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app',
                'tuscany-domain-manager', 'tuscany-sca-all', 'webmin',
                'WebProxyPortlet',
                'whiteboard', 'activemq', 'apacheds', 'avro-tools',
                'css-validator',
                'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws',
                'force-oauth',
                'groovysoap-all-jsr06', 'jackrabbit-standalone',
                'jetty-webapp', 'jftp',
                'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all',
                'ocpsoft-pretty-faces',
                'org.apache.felix.webconsole', 'org.apache.sling.openidauth',
                'org.jbundle.util.webapp.redirect',
                'org.talend.esb.job.console',
                'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'}

    input_bugs = set()
    input_bugs |= sql_bugs
    input_bugs |= xss_bugs

    total_projects = len(projects)
    count = 0

    workbook = xlsxwriter.Workbook('bug_sources.xlsx')
    worksheet = workbook.add_worksheet()
    row = 0

    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(),
                                     fields=['JarMetadata.group_id',
                                             'JarMetadata.artifact_id',
                                             'JarMetadata.version',
                                             'JarMetadata.version_order',
                                             'BugCollection.BugInstance.category',
                                             'BugCollection.BugInstance.type',
                                             'BugCollection.BugInstance.SourceLine.classname',
                                             'BugCollection.BugInstance.SourceLine.start',
                                             'BugCollection.BugInstance.SourceLine.end'])
        doc_list = piter.documents_list()
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (
            count, total_projects, p.group_id(), p.artifact_id(),
            len(doc_list))

        for d in doc_list:
            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    # print 'Invalid BugInstance (%s)' % (bi,)
                    continue

                bug_c = bi.get('category', '')
                if bug_c == 'SECURITY':
                    bug_type = bi.get('type', None)

                    if bug_type is None:
                        print 'Invalid Type!'
                        continue

                    if bug_type in security_bugs:
                        if p.artifact_id() in input_bugs:
                            col = 0
                            source = bi.get('SourceLine', {})
                            worksheet.write(row, col, p.artifact_id())
                            if isinstance(source, list):
                                for i, j in enumerate(source):
                                    worksheet.write(row, col + 1 + i,
                                                    j.get('classname',
                                                          'NotSet'))
                                    worksheet.write(row, col + 2 + i,
                                                    j.get('start', 'NotSet'))
                                    worksheet.write(row, col + 3 + i,
                                                    j.get('end', 'NotSet'))
                            elif isinstance(source, dict):
                                worksheet.write(row, col + 1,
                                                source.get('classname',
                                                           'NotSet'))
                                worksheet.write(row, col + 2,
                                                source.get('start', 'NotSet'))
                                worksheet.write(row, col + 3,
                                                source.get('end', 'NotSet'))
                            row += 1

        print row
def main():
    projects = load_evolution_projects_json()
    results = OrderedDict()
    total_projects = len(projects)
    security_bugs = ['HRS_REQUEST_PARAMETER_TO_COOKIE',
                     'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER',
                     'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
                     'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
                     'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
                     'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
                     'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER']

    sql_bugs = {'activemq-all', 'activemq', 'activeobjects', 'cas-workflow',
                'ebxmlms', 'efaps-kernel', 'fabric3-binding-ws', 'geotk-metadata-sql',
                'jackrabbit-standalone', 'james', 'james-server-mailets', 'jcaptcha-all',
                'jdatabaseimport', 'jetty-webapp', 'jonas-jms-manager', 'joram', 'kernel',
                'makumba', 'MetaModel', 'nunaliit2-adhocQueries', 'openjms',
                'org.openl.rules.eclipse.ui.wizard', 'sandesha2-persistence',
                'servicemix-components', 'sesame', 'sonar-application', 'sqltool',
                'sqltool-j5', 'squirrel-sql', 'torque', 'transactions-jta',
                'ujo-orm', 'xmlui'}

    xss_bugs = {'activemq-all', 'activemq-web', 'makumba', 'netcdf', 'opendap',
                'org.talend.esb.job.console', 'rdfbean-sparql', 'tika-app',
                'tuscany-domain-manager', 'tuscany-sca-all', 'webmin', 'WebProxyPortlet',
                'whiteboard', 'activemq', 'apacheds', 'avro-tools', 'css-validator',
                'dspace-jspui-api', 'dspace-lni-core', 'fabric3-binding-ws', 'force-oauth',
                'groovysoap-all-jsr06', 'jackrabbit-standalone', 'jetty-webapp', 'jftp',
                'makumba', 'MessAdmin-Core', 'myfaces', 'myfaces-all', 'ocpsoft-pretty-faces',
                'org.apache.felix.webconsole', 'org.apache.sling.openidauth',
                'org.jbundle.util.webapp.redirect', 'org.talend.esb.job.console',
                'pustefix-webservices-jaxws', 'sonar-application', 'vt-ldap'}

    input_bugs = set()
    input_bugs |= sql_bugs
    input_bugs |= xss_bugs

    count = 0

    print 'Found %d Projects' % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(p.group_id(), p.artifact_id(), fields=['JarMetadata.group_id', 'JarMetadata.artifact_id', 'JarMetadata.version', 'JarMetadata.version_order', 'BugCollection.BugInstance.category', 'BugCollection.BugInstance.type', 'BugCollection.BugInstance.Class.classname','BugCollection.BugInstance.Method.name', 'BugCollection.BugInstance.Field.name'])
        doc_list = piter.documents_list()
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))

        for d in doc_list:
            if d['JarMetadata']['version_order'] == 0:
                continue

            proj_array_count = ArrayCount()
            signatures = []

            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    #print 'Invalid BugInstance (%s)' % (bi,)
                    continue

                bug_c = bi.get('category', '')
                if bug_c == 'SECURITY':
                    bug_type = bi.get('type', None)
                    
                    if bug_type is None:
                        print 'Invalid Type!'
                        continue

                    if bug_type in security_bugs:
                        if p.artifact_id() in input_bugs:
                            bug_category = 'INPUT_VALIDATION_BUGS'
                        else:
                            continue
                    else:
                        bug_category = 'SECURITY_REST'
                else:
                    bug_category = bug_c

                # create signature
                signatures_ids = []
                classnames = bi['Class']

                if isinstance(classnames, list):
                    for c in classnames:
                        signatures_ids.append(c.get('classname', 'NotSet'))
                elif isinstance(classnames, dict):
                    signatures_ids.append(classnames.get('classname', 'NotSet'))

                # methods
                methodnames = bi.get('Method', {})

                if isinstance(methodnames, list):
                    for m in methodnames:
                        signatures_ids.append(m.get('name', 'NotSet'))
                elif isinstance(methodnames, dict):
                    signatures_ids.append(methodnames.get('name', 'NotSet'))

                # fields
                fieldnames = bi.get('Field', {})
                if isinstance(fieldnames, list):
                    for f in fieldnames:
                        signatures_ids.append(f.get('name', 'NotSet'))
                elif isinstance(fieldnames, dict):
                    signatures_ids.append(fieldnames.get('name', 'NotSet'))

                bug_type = bi['type']
                signature = '%s||%s||%s' % (bug_category, bug_type, '||'.join(signatures_ids))
                signatures.append(signature)
                proj_array_count.incr('bug_category')

            results['%s||%s||%s' % (p.group_id(), p.artifact_id(), d['JarMetadata']['version'])] = {'Counters': proj_array_count.get_series(), 'Bugs': signatures, 'version_order': d['JarMetadata']['version_order']}

    save_to_file('data/bug_persistence.json', json.dumps(results))
Beispiel #9
0
def main():
    projects = load_evolution_projects_json()
    results = {}
    security_bugs = [
        'HRS_REQUEST_PARAMETER_TO_COOKIE',
        'HRS_REQUEST_PARAMETER_TO_HTTP_HEADER', 'PT_ABSOLUTE_PATH_TRAVERSAL',
        'SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE',
        'SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING',
        'XSS_REQUEST_PARAMETER_TO_JSP_WRITER',
        'XSS_REQUEST_PARAMETER_TO_SEND_ERROR',
        'XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER'
    ]
    total_projects = len(projects)
    count = 0

    print 'Found %d Projects' % (total_projects, )

    for p in projects:
        piter = MongoProjectIterator(
            p.group_id(),
            p.artifact_id(),
            fields=[
                'JarMetadata.group_id', 'JarMetadata.artifact_id',
                'JarMetadata.version', 'JarMetadata.version_order',
                'BugCollection.BugInstance.category',
                'BugCollection.BugInstance.type',
                'BugCollection.BugInstance.Class.classname',
                'BugCollection.BugInstance.Method.name',
                'BugCollection.BugInstance.Field.name'
            ])
        doc_list = piter.documents_list()
        proj_array_count = ArrayCount()
        bug_list = []
        count += 1

        print '[%d:%d] %s||%s: %d versions' % (count, total_projects,
                                               p.group_id(), p.artifact_id(),
                                               len(doc_list))

        for d in doc_list:
            for bi in d.get('BugCollection', {}).get('BugInstance', []):
                if not isinstance(bi, dict):
                    #print 'Invalid BugInstance (%s)' % (bi,)
                    continue

                bug_c = bi.get('category', '')
                if bug_c == 'SECURITY':
                    bug_type = bi.get('type', None)

                    if bug_type is None:
                        print 'Invalid Type!'
                        continue

                    if bug_type in security_bugs:
                        bug_category = 'SECURITY_HIGH'
                    else:
                        bug_category = 'SECURITY_LOW'
                else:
                    bug_category = bug_c

                # create signature
                signatures_ids = []
                classnames = bi['Class']

                if isinstance(classnames, list):
                    for c in classnames:
                        signatures_ids.append(c.get('classname', 'NotSet'))
                elif isinstance(classnames, dict):
                    signatures_ids.append(classnames.get(
                        'classname', 'NotSet'))

                # methods
                methodnames = bi.get('Method', {})

                if isinstance(methodnames, list):
                    for m in methodnames:
                        signatures_ids.append(m.get('name', 'NotSet'))
                elif isinstance(methodnames, dict):
                    signatures_ids.append(methodnames.get('name', 'NotSet'))

                # fields
                fieldnames = bi.get('Field', {})
                if isinstance(fieldnames, list):
                    for f in fieldnames:
                        signatures_ids.append(f.get('name', 'NotSet'))
                elif isinstance(fieldnames, dict):
                    signatures_ids.append(fieldnames.get('name', 'NotSet'))

                type = bi['type']
                signature = '%s||%s||%s' % (bug_category, type,
                                            '||'.join(signatures_ids))

                # method
                if signature not in bug_list:
                    bug_list.append(signature)
                    proj_array_count.incr(bug_category)

                proj_array_count.incr('TOTAL_' + bug_category)

        print proj_array_count.get_series()
        results['%s||%s' % (p.group_id(),
                            p.artifact_id())] = proj_array_count.get_series()

    save_to_file('bug_correlation_counters.json', json.dumps(results))
def main():
    projects = load_evolution_projects_json()
    results = {}
    security_bugs = [
        "HRS_REQUEST_PARAMETER_TO_COOKIE",
        "HRS_REQUEST_PARAMETER_TO_HTTP_HEADER",
        "SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE",
        "SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING",
        "XSS_REQUEST_PARAMETER_TO_JSP_WRITER",
        "XSS_REQUEST_PARAMETER_TO_SEND_ERROR",
        "XSS_REQUEST_PARAMETER_TO_SERVLET_WRITER",
    ]

    sql_bugs = {
        "activemq-all",
        "activemq",
        "activeobjects",
        "cas-workflow",
        "ebxmlms",
        "efaps-kernel",
        "fabric3-binding-ws",
        "geotk-metadata-sql",
        "jackrabbit-standalone",
        "james",
        "james-server-mailets",
        "jcaptcha-all",
        "jdatabaseimport",
        "jetty-webapp",
        "jonas-jms-manager",
        "joram",
        "kernel",
        "makumba",
        "MetaModel",
        "nunaliit2-adhocQueries",
        "openjms",
        "org.openl.rules.eclipse.ui.wizard",
        "sandesha2-persistence",
        "servicemix-components",
        "sesame",
        "sonar-application",
        "sqltool",
        "sqltool-j5",
        "squirrel-sql",
        "torque",
        "transactions-jta",
        "ujo-orm",
        "xmlui",
    }

    xss_bugs = {
        "activemq-all",
        "activemq-web",
        "makumba",
        "netcdf",
        "opendap",
        "org.talend.esb.job.console",
        "rdfbean-sparql",
        "tika-app",
        "tuscany-domain-manager",
        "tuscany-sca-all",
        "webmin",
        "WebProxyPortlet",
        "whiteboard",
        "activemq",
        "apacheds",
        "avro-tools",
        "css-validator",
        "dspace-jspui-api",
        "dspace-lni-core",
        "fabric3-binding-ws",
        "force-oauth",
        "groovysoap-all-jsr06",
        "jackrabbit-standalone",
        "jetty-webapp",
        "jftp",
        "makumba",
        "MessAdmin-Core",
        "myfaces",
        "myfaces-all",
        "ocpsoft-pretty-faces",
        "org.apache.felix.webconsole",
        "org.apache.sling.openidauth",
        "org.jbundle.util.webapp.redirect",
        "org.talend.esb.job.console",
        "pustefix-webservices-jaxws",
        "sonar-application",
        "vt-ldap",
    }

    input_bugs = set()
    input_bugs |= sql_bugs
    input_bugs |= xss_bugs

    total_projects = len(projects)
    count = 0

    print "Found %d Projects" % (total_projects,)

    for p in projects:
        piter = MongoProjectIterator(
            p.group_id(),
            p.artifact_id(),
            fields=[
                "JarMetadata.group_id",
                "JarMetadata.artifact_id",
                "JarMetadata.version",
                "JarMetadata.version_order",
                "BugCollection.BugInstance.category",
                "BugCollection.BugInstance.type",
                "BugCollection.BugInstance.Class.classname",
                "BugCollection.BugInstance.Method.name",
                "BugCollection.BugInstance.Field.name",
            ],
        )
        doc_list = piter.documents_list()
        proj_array_count = ArrayCount()
        bug_list = []
        count += 1

        print "[%d:%d] %s||%s: %d versions" % (count, total_projects, p.group_id(), p.artifact_id(), len(doc_list))

        for d in doc_list:
            for bi in d.get("BugCollection", {}).get("BugInstance", []):
                if not isinstance(bi, dict):
                    # print 'Invalid BugInstance (%s)' % (bi,)
                    continue

                bug_c = bi.get("category", "")
                if bug_c == "SECURITY":
                    bug_type = bi.get("type", None)

                    if bug_type is None:
                        print "Invalid Type!"
                        continue

                    if bug_type in security_bugs:
                        if p.artifact_id() in input_bugs:
                            bug_category = "INPUT_VALIDATION_BUGS"
                        else:
                            continue
                    else:
                        bug_category = "SECURITY_REST"
                else:
                    bug_category = bug_c

                # create signature
                signatures_ids = []
                classnames = bi["Class"]

                if isinstance(classnames, list):
                    for c in classnames:
                        signatures_ids.append(c.get("classname", "NotSet"))
                elif isinstance(classnames, dict):
                    signatures_ids.append(classnames.get("classname", "NotSet"))

                # methods
                methodnames = bi.get("Method", {})

                if isinstance(methodnames, list):
                    for m in methodnames:
                        signatures_ids.append(m.get("name", "NotSet"))
                elif isinstance(methodnames, dict):
                    signatures_ids.append(methodnames.get("name", "NotSet"))

                # fields
                fieldnames = bi.get("Field", {})
                if isinstance(fieldnames, list):
                    for f in fieldnames:
                        signatures_ids.append(f.get("name", "NotSet"))
                elif isinstance(fieldnames, dict):
                    signatures_ids.append(fieldnames.get("name", "NotSet"))

                type = bi["type"]
                signature = "%s||%s||%s" % (bug_category, type, "||".join(signatures_ids))

                # method
                if signature not in bug_list:
                    bug_list.append(signature)
                    proj_array_count.incr(bug_category)

                proj_array_count.incr("TOTAL_" + bug_category)

        print proj_array_count.get_series()
        results["%s||%s" % (p.group_id(), p.artifact_id())] = proj_array_count.get_series()

    save_to_file("data/bug_correlation_counters.json", json.dumps(results))