예제 #1
0
 def test_check_if_website_exist_done(self, value, expected):
     self.website.status = Website.Status.DONE
     db.session.add(self.website)
     db.session.commit()
     website_backend = WebsiteBackend()
     self.assertEquals(website_backend._check_if_website_exist_done(value),
                       expected)  # noqa
예제 #2
0
    def test_save_website_links_website_nosql_not_found(self):
        self.website.status = Website.Status.DONE
        db.session.add(self.website)
        db.session.commit()
        with self.assertRaises(WebsiteBackendException) as error:
            list_urls = ['http://ibm.com.br']
            website_backend = WebsiteBackend()
            website_backend._save_website_links(self.website, list_urls)

        self.assertEqual(str(error.exception), "Website NoSql not found")
예제 #3
0
    def test_get_available_links_without_response_text(self):
        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_get_available_links_without_response_text.yaml'
        ):  # noqa

            with self.assertRaises(WebsiteBackendException) as error:
                website_backend = WebsiteBackend()
                website_backend._get_available_links(self.website)

            self.assertEqual(str(error.exception),
                             "Response text without value")  # noqa
예제 #4
0
    def test_get_available_links_with_connection_error(self):
        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_get_available_links_with_connection_error.yaml'
        ):  # noqa
            self.website.url = 'http://a.com'

            with self.assertRaises(WebsiteBackendException) as error:
                website_backend = WebsiteBackend()
                website_backend._get_available_links(self.website)

            self.assertEqual(str(error.exception),
                             "ConnectionError on request")  # noqa
예제 #5
0
    def test_get_available_links_without_status_200(self):
        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_get_available_links_without_status_200.yaml'
        ):  # noqa
            self.website.url = 'http://vtrmantovani.com.br/not-found'

            with self.assertRaises(WebsiteBackendException) as error:
                website_backend = WebsiteBackend()
                website_backend._get_available_links(self.website)

            self.assertEqual(str(error.exception),
                             "Response status code not is 200")  # noqa
예제 #6
0
    def test_get_available_links_without_urls(self):
        db.session.add(self.website)
        db.session.commit()

        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_get_available_links_without_urls.yaml'
        ):  # noqa
            website_backend = WebsiteBackend()
            list_urls = website_backend._get_available_links(self.website)

        self.assertEquals(list_urls, [])
        self.assertEqual(Website.query.count(), 1)
예제 #7
0
    def test_save_website_links_without_website_done(self):
        db.session.add(self.website)
        db.session.commit()
        list_urls = ['http://ibm.com.br']

        website_backend = WebsiteBackend()
        website_backend._save_website_links(self.website, list_urls)
        webiste_nosql = WebsiteNoSql()

        self.assertEqual(Website.query.count(), 2)
        self.assertTrue(webiste_nosql.get(1))
        webiste_nosql.delete(1)
예제 #8
0
    def test_failed_save_website_available_links(self):
        db.session.add(self.website)
        db.session.commit()

        website_backend = WebsiteBackend()
        website_backend.failed_save_website_available_links(
            self.website, 'Error')  # noqa

        website_db = Website.query.filter(Website.id == 1).first()  # noqa
        self.assertEqual(Website.query.count(), 1)
        self.assertEqual(website_db.status.value, 'ERROR')
        self.assertEqual(website_db.status_desc, 'Error')  # noqa
예제 #9
0
    def test_save_website_available_links(self, mock_save_website):
        db.session.add(self.website)
        db.session.commit()

        self.assertEqual(Website.query.count(), 1)

        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_save_website_available_links.yaml'
        ):  # noqa
            website_backend = WebsiteBackend()
            website_backend.save_website_available_links(self.website)

        website_db = Website.query.filter(Website.id == 1).first()  # noqa

        self.assertEqual(website_db.status.value, "DONE")
예제 #10
0
    def test_get_available_links(self):
        db.session.add(self.website)
        db.session.commit()

        self.assertEqual(Website.query.count(), 1)

        with vcr.use_cassette(
                'tests/fixtures/cassettes/test_get_available_links.yaml'
        ):  # noqa
            website_backend = WebsiteBackend()
            list_urls = website_backend._get_available_links(self.website)

        self.assertEquals(list_urls[0], 'https://github.com/vtrmantovani')
        self.assertEquals(list_urls[1],
                          'https://www.linkedin.com/in/vtrmantovani')  # noqa
예제 #11
0
    def test_save_website_links_without_website_nosql_differents_urls(self):
        self.website.status = Website.Status.DONE
        db.session.add(self.website)
        db.session.commit()

        webiste_nosql = WebsiteNoSql()
        list_urls = ['http://vtrmantovani.com.br']
        body = {'urls': list_urls}
        webiste_nosql.create(1, body)

        website_backend = WebsiteBackend()
        website_backend._save_website_links(self.website, list_urls)

        self.assertEqual(Website.query.count(), 1)
        self.assertTrue(webiste_nosql.get(1))
        webiste_nosql.delete(1)
예제 #12
0
def process_available_links(self, website_id):

    logger.info(
        "Task to process available links: started. website_id={0}".format(
            website_id))  # noqa

    try:
        website = Website.query.filter(Website.id == website_id).first()

        if not website:
            logger.error(
                "Task to process available links: website not found. website_id={0}"
                .format(website_id))  # noqa
            raise TasksException(
                'Website not found: website_id={0}'.format(website_id))  # noqa

        WebsiteBackend().save_website_available_links(website)

    except ConnectTimeout as exc:
        self.retry(exc=exc, countdown=180)
    except SQLAlchemyError as exc:
        db.session.rollback()
        self.retry(exc=exc, countdown=180)
    except TasksException as e:
        raise TasksException(str(e))
    except Exception as e:
        logger.exception(
            "Task to process available links: exception. website_id={0}".
            format(website_id))  # noqa

        if website:
            WebsiteBackend().failed_save_website_available_links(
                website, str(e))  # noqa

    logger.info(
        "Task to process available links: ended. website_id={0}".format(
            website_id))  # noqa
예제 #13
0
    def test_save_website(self):
        website_backend = WebsiteBackend()
        website_backend._save_website("http://vtrmantovani.com.br")

        self.assertEqual(Website.query.count(), 1)
예제 #14
0
 def test_put_website_links_on_quee_with_invalid_url(self):
     list_urls = ['http://ibm']
     website_backend = WebsiteBackend()
     website_backend._put_website_links_on_quee(list_urls)
     self.assertEqual(Website.query.count(), 0)
예제 #15
0
 def test_put_website_links_on_quee(self):
     list_urls = ['http://ibm.com.br']
     website_backend = WebsiteBackend()
     website_backend._put_website_links_on_quee(list_urls)
     self.assertEqual(Website.query.count(), 1)