Python MultiProcessingDocumentParser 예제들

예제 #1

파일: test_mp_document_parser.py 프로젝트: xiaofengtongxue/w3af

class TestMPDocumentParser(unittest.TestCase):
    def setUp(self):
        self.url = URL('http://w3af.com')
        self.headers = Headers([(u'content-type', u'text/html')])
        self.mpdoc = MultiProcessingDocumentParser()

    def tearDown(self):
        self.mpdoc.stop_workers()

    def test_basic(self):
        resp = HTTPResponse(200, '<a href="/abc">hello</a>', self.headers,
                            self.url, self.url)

        parser = self.mpdoc.get_document_parser_for(resp)

        parsed_refs, _ = parser.get_references()
        self.assertEqual([URL('http://w3af.com/abc')], parsed_refs)

    def test_no_parser_for_images(self):
        body = ''
        url = URL('http://w3af.com/foo.jpg')
        headers = Headers([(u'content-type', u'image/jpeg')])
        resp = HTTPResponse(200, body, headers, url, url)

        try:
            self.mpdoc.get_document_parser_for(resp)
        except Exception, e:
            self.assertEqual(str(e), 'There is no parser for images.')
        else:

예제 #2

파일: test_mp_document_parser.py 프로젝트: batmanWjw/w3af

class TestMPDocumentParser(unittest.TestCase):

    def setUp(self):
        self.url = URL('http://w3af.com')
        self.headers = Headers([(u'content-type', u'text/html')])
        self.mpdoc = MultiProcessingDocumentParser()

    def tearDown(self):
        self.mpdoc.stop_workers()

    def test_basic(self):
        resp = HTTPResponse(200, '<a href="/abc">hello</a>',
                            self.headers, self.url, self.url)

        parser = self.mpdoc.get_document_parser_for(resp)

        parsed_refs, _ = parser.get_references()
        self.assertEqual([URL('http://w3af.com/abc')], parsed_refs)

    def test_no_parser_for_images(self):
        body = ''
        url = URL('http://w3af.com/foo.jpg')
        headers = Headers([(u'content-type', u'image/jpeg')])
        resp = HTTPResponse(200, body, headers, url, url)

        try:
            self.mpdoc.get_document_parser_for(resp)
        except Exception, e:
            self.assertEqual(str(e), 'There is no parser for images.')
        else:

예제 #3

파일: test_mp_document_parser.py 프로젝트: xiaofengtongxue/w3af

def daemon_child(queue):
    dpc = MultiProcessingDocumentParser()

    try:
        dpc.start_workers()
    except AssertionError:
        queue.put(True)
    else:
        queue.put(False)

예제 #4

파일: test_mp_document_parser.py 프로젝트: batmanWjw/w3af

def daemon_child(queue):
    dpc = MultiProcessingDocumentParser()

    try:
        dpc.start_workers()
    except AssertionError:
        queue.put(True)
    else:
        queue.put(False)

예제 #5

파일: test_mp_document_parser.py 프로젝트: xiaofengtongxue/w3af

 def setUp(self):
     self.url = URL('http://w3af.com')
     self.headers = Headers([(u'content-type', u'text/html')])
     self.mpdoc = MultiProcessingDocumentParser()

예제 #6

파일: test_mp_document_parser.py 프로젝트: hougomartim/w3af

class TestMPDocumentParser(unittest.TestCase):

    def setUp(self):
        self.url = URL('http://w3af.com')
        self.headers = Headers([(u'content-type', u'text/html')])
        self.mpdoc = MultiProcessingDocumentParser()

    def tearDown(self):
        self.mpdoc.stop_workers()

    def test_basic(self):
        resp = HTTPResponse(200, '<a href="/abc">hello</a>',
                            self.headers, self.url, self.url)

        parser = self.mpdoc.get_document_parser_for(resp)

        parsed_refs, _ = parser.get_references()
        self.assertEqual([URL('http://w3af.com/abc')], parsed_refs)

    def test_no_parser_for_images(self):
        body = ''
        url = URL('http://w3af.com/foo.jpg')
        headers = Headers([(u'content-type', u'image/jpeg')])
        resp = HTTPResponse(200, body, headers, url, url)

        try:
            self.mpdoc.get_document_parser_for(resp)
        except Exception as e:
            self.assertEqual(str(e), 'There is no parser for images.')
        else:
            self.assertTrue(False, 'Expected exception!')

    def test_parser_timeout(self):
        """
        Test to verify fix for https://github.com/andresriancho/w3af/issues/6723
        "w3af running long time more than 24h"
        """
        mmpdp = 'w3af.core.data.parsers.mp_document_parser.%s'
        kmpdp = mmpdp % 'MultiProcessingDocumentParser.%s'
        modp = 'w3af.core.data.parsers.document_parser.%s'

        with patch(mmpdp % 'om.out') as om_mock,\
             patch(kmpdp % 'PARSER_TIMEOUT', new_callable=PropertyMock) as timeout_mock,\
             patch(kmpdp % 'MAX_WORKERS', new_callable=PropertyMock) as max_workers_mock,\
             patch(modp % 'DocumentParser.PARSERS', new_callable=PropertyMock) as parsers_mock:

            #
            #   Test the timeout
            #
            html = '<html>DelayedParser!</html>'
            http_resp = _build_http_response(html, u'text/html')

            timeout_mock.return_value = 1
            max_workers_mock.return_value = 1
            parsers_mock.return_value = [DelayedParser, HTMLParser]

            try:
                self.mpdoc.get_document_parser_for(http_resp)
            except TimeoutError, toe:
                self._is_timeout_exception_message(toe, om_mock, http_resp)
            else:

예제 #7

파일: test_mp_document_parser.py 프로젝트: batmanWjw/w3af

 def setUp(self):
     self.url = URL('http://w3af.com')
     self.headers = Headers([(u'content-type', u'text/html')])
     self.mpdoc = MultiProcessingDocumentParser()