Example #1
0
    def enqueue(self, urls):
        if not urls:
            return

        for url, score in urls:
            Page.add_page(self.db, self.cache, url, score, self.async_get_func,
                          self.publish, self.config, self.girl,
                          self.violation_definitions, self.handle_page_added)

        self.wait_for_async_requests()
Example #2
0
    def post(self):
        post_data = loads(self.request.body)
        url = post_data["url"]
        score = float(post_data.get("score", self.application.config.DEFAULT_PAGE_SCORE))

        result = yield Page.add_page(
            self.db,
            self.application.cache,
            url,
            score,
            self.application.http_client.fetch,
            self.application.event_bus.publish,
            self.application.config,
        )

        created, url, result = result

        if not created and result["reason"] == "invalid_url":
            self.set_status(400, "Invalid url [%s]" % url)
            self.write_json(
                {"reason": "invalid_url", "url": url, "status": result["status"], "details": result["details"]}
            )
            return

        if not created and result["reason"] == "redirect":
            self.set_status(400, "Redirect URL [%s]" % url)
            self.write_json({"reason": "redirect", "url": url, "effectiveUrl": result["effectiveUrl"]})
            return

        self.write(str(result))
        self.finish()
Example #3
0
    def enqueue(self, urls):
        if not urls:
            return

        for url, score in urls:
            Page.add_page(
                self.db,
                self.cache,
                url,
                score,
                self.async_get_func,
                self.publish,
                self.config,
                self.handle_page_added
            )

        self.wait_for_async_requests()
Example #4
0
    def post(self):
        post_data = loads(self.request.body)
        url = post_data['url']
        score = float(
            post_data.get('score', self.application.config.DEFAULT_PAGE_SCORE))

        result = yield Page.add_page(
            self.db, self.application.cache, url, score,
            self.application.http_client.fetch,
            self.application.event_bus.publish, self.application.config,
            self.application.girl, self.application.default_violations_values,
            self.application.violation_definitions)

        created, url, result = result

        if not created and result['reason'] == 'invalid_url':
            self.set_status(400, self._('Invalid url [%s]') % url)
            self.write_json({
                'reason': 'invalid_url',
                'url': url,
                'status': result['status'],
                'details': result['details']
            })
            return

        if not created and result['reason'] == 'redirect':
            self.set_status(400,
                            self._('Supplied URL is a redirect [%s]') % url)
            self.write_json({
                'reason': 'redirect',
                'url': url,
                'effectiveUrl': result['effectiveUrl']
            })
            return

        yield self.application.cache.add_next_job_bucket(result, url)

        self.write(str(result))
        self.finish()
Example #5
0
    def post(self):
        post_data = loads(self.request.body)
        url = post_data['url']
        score = float(post_data.get('score', self.application.config.DEFAULT_PAGE_SCORE))

        result = yield Page.add_page(
            self.db,
            self.application.cache,
            url,
            score,
            self.application.http_client.fetch,
            self.application.event_bus.publish,
            self.application.config
        )

        created, url, result = result

        if not created and result['reason'] == 'invalid_url':
            self.set_status(400, 'Invalid url [%s]' % url)
            self.write_json({
                'reason': 'invalid_url',
                'url': url,
                'status': result['status'],
                'details': result['details']
            })
            return

        if not created and result['reason'] == 'redirect':
            self.set_status(400, 'Redirect URL [%s]' % url)
            self.write_json({
                'reason': 'redirect',
                'url': url,
                'effectiveUrl': result['effectiveUrl']
            })
            return

        self.write(str(result))
        self.finish()
Example #6
0
    def post(self):
        post_data = loads(self.request.body)
        url = post_data["url"]
        score = float(post_data.get("score", self.application.config.DEFAULT_PAGE_SCORE))

        result = yield Page.add_page(
            self.db,
            self.application.cache,
            url,
            score,
            self.application.http_client.fetch,
            self.application.event_bus.publish,
            self.application.config,
            self.application.girl,
            self.application.default_violations_values,
            self.application.violation_definitions,
        )

        created, url, result = result

        if not created and result["reason"] == "invalid_url":
            self.set_status(400, self._("Invalid url [%s]") % url)
            self.write_json(
                {"reason": "invalid_url", "url": url, "status": result["status"], "details": result["details"]}
            )
            return

        if not created and result["reason"] == "redirect":
            self.set_status(400, self._("Supplied URL is a redirect [%s]") % url)
            self.write_json({"reason": "redirect", "url": url, "effectiveUrl": result["effectiveUrl"]})
            return

        yield self.application.cache.add_next_job_bucket(result, url)

        self.write(str(result))
        self.finish()