Exemple #1
0
    def _download_image(self) -> Tuple[str, str]:
        """Downloads an image into the URL cache.
        Returns:
            A (host, media_id) tuple representing the MXC URI of the image.
        """
        self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]

        channel = self.make_request(
            "GET",
            "preview_url?url=http://cdn.twitter.com/matrixdotorg",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: image/png\r\n\r\n"
            % (len(SMALL_PNG), ) + SMALL_PNG)

        self.pump()
        self.assertEqual(channel.code, 200)
        body = channel.json_body
        mxc_uri = body["og:image"]
        host, _port, media_id = parse_and_validate_mxc_uri(mxc_uri)
        self.assertIsNone(_port)
        return host, media_id
Exemple #2
0
def applyPatch(patch, patchLevel="0", reactor=None):
    """
    Apply a patch to the current git repository.

    @param patch: Patch to apply
    @type patch: L{str}

    @param patchLevel: Number of directries to strip from paths in patch
    """
    proto = AccumulatingProtocol()
    done = Deferred()
    proto.closedDeferred = done

    def feedPatch(proto):
        proto.transport.write(patch)
        proto.transport.closeStdin()

    connectProtocol(
        ProcessEndpoint(reactor, "git",
                        ("git", "apply", "--index", "-p", patchLevel)),
        proto).addCallback(feedPatch)

    def eb(_):
        # Note, we can't print why git apply failed due to https://tm.tl/#6576
        proto.closedReason.trap(ConnectionDone)

    done.addCallback(eb)
    return done
Exemple #3
0
    def test_nonexistent_image(self) -> None:
        """If the preview image doesn't exist, ensure some data is returned."""
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        end_content = (
            b"""<html><body><img src="http://cdn.matrix.org/foo.jpg"></body></html>"""
        )

        channel = self.make_request(
            "GET",
            "preview_url?url=http://matrix.org",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
             b'Content-Type: text/html; charset="utf8"\r\n\r\n') %
            (len(end_content), ) + end_content)

        self.pump()
        self.assertEqual(channel.code, 200)

        # The image should not be in the result.
        self.assertNotIn("og:image", channel.json_body)
Exemple #4
0
    def test_ipaddr(self):
        """
        IP addresses can be previewed directly.
        """
        self.lookups["example.com"] = [(IPv4Address, "8.8.8.8")]

        request, channel = self.make_request(
            "GET", "url_preview?url=http://example.com", shorthand=False)
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content), ) + self.end_content)

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(channel.json_body, {
            "og:title": "~matrix~",
            "og:description": "hi"
        })
Exemple #5
0
    def test_overlong_title(self):
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        end_content = (b"<html><head>"
                       b"<title>" + b"x" * 2000 + b"</title>"
                       b'<meta property="og:description" content="hi" />'
                       b"</head></html>")

        channel = self.make_request(
            "GET",
            "preview_url?url=http://matrix.org",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
             b'Content-Type: text/html; charset="windows-1251"\r\n\r\n') %
            (len(end_content), ) + end_content)

        self.pump()
        self.assertEqual(channel.code, 200)
        res = channel.json_body
        # We should only see the `og:description` field, as `title` is too long and should be stripped out
        self.assertCountEqual(["og:description"], res.keys())
Exemple #6
0
    def test_inline_data_url(self) -> None:
        """
        An inline image (as a data URL) should be parsed properly.
        """
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        data = base64.b64encode(SMALL_PNG)

        end_content = (b"<html><head>"
                       b'<img src="data:image/png;base64,%s" />'
                       b"</head></html>") % (data, )

        channel = self.make_request(
            "GET",
            "preview_url?url=http://matrix.org",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
             b'Content-Type: text/html; charset="utf8"\r\n\r\n') %
            (len(end_content), ) + end_content)

        self.pump()
        self.assertEqual(channel.code, 200)
        self._assert_small_png(channel.json_body)
Exemple #7
0
    def test_non_ascii_preview_httpequiv(self):
        self.lookups["matrix.org"] = [(IPv4Address, "8.8.8.8")]

        end_content = (
            b'<html><head>'
            b'<meta http-equiv="Content-Type" content="text/html; charset=windows-1251"/>'
            b'<meta property="og:title" content="\xe4\xea\xe0" />'
            b'<meta property="og:description" content="hi" />'
            b'</head></html>')

        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False)
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
             b"Content-Type: text/html; charset=\"utf8\"\r\n\r\n") %
            (len(end_content), ) + end_content)

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(channel.json_body["og:title"], u"\u0434\u043a\u0430")
    def test_blacklisted_ip_range_whitelisted_ip(self):
        """
        Blacklisted but then subsequently whitelisted IP addresses can be
        spidered.
        """
        self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")]

        request, channel = self.make_request(
            "GET", "url_preview?url=http://example.com", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)

        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))

        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content),)
            + self.end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )
    def test_blacklisted_ip_range_whitelisted_ip(self):
        """
        Blacklisted but then subsequently whitelisted IP addresses can be
        spidered.
        """
        self.lookups["example.com"] = [(IPv4Address, "1.1.1.1")]

        channel = self.make_request(
            "GET",
            "preview_url?url=http://example.com",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)

        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))

        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content),)
            + self.end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )
    def test_non_ascii_preview_content_type(self):
        self.lookups["matrix.org"] = [(IPv4Address, "8.8.8.8")]

        end_content = (
            b'<html><head>'
            b'<meta property="og:title" content="\xe4\xea\xe0" />'
            b'<meta property="og:description" content="hi" />'
            b'</head></html>'
        )

        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (
                b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                b"Content-Type: text/html; charset=\"windows-1251\"\r\n\r\n"
            )
            % (len(end_content),)
            + end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(channel.json_body["og:title"], u"\u0434\u043a\u0430")
Exemple #11
0
    def test_audio_rejected(self) -> None:
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        end_content = b"anything"

        channel = self.make_request(
            "GET",
            "preview_url?url=http://matrix.org",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived((b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                             b"Content-Type: audio/aac\r\n\r\n") %
                            (len(end_content)) + end_content)

        self.pump()
        self.assertEqual(channel.code, 502)
        self.assertEqual(
            channel.json_body,
            {
                "errcode":
                "M_UNKNOWN",
                "error":
                "Requested file's content type not allowed for this operation: audio/aac",
            },
        )
    def test_non_ascii_preview_content_type(self):
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        end_content = (
            b"<html><head>"
            b'<meta property="og:title" content="\xe4\xea\xe0" />'
            b'<meta property="og:description" content="hi" />'
            b"</head></html>"
        )

        channel = self.make_request(
            "GET",
            "preview_url?url=http://matrix.org",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (
                b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                b'Content-Type: text/html; charset="windows-1251"\r\n\r\n'
            )
            % (len(end_content),)
            + end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(channel.json_body["og:title"], "\u0434\u043a\u0430")
    def test_cache_returns_correct_type(self):
        self.lookups["matrix.org"] = [(IPv4Address, "8.8.8.8")]

        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content),)
            + self.end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )

        # Check the cache returns the correct response
        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        # Check the cache response has the same content
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )

        # Clear the in-memory cache
        self.assertIn("http://matrix.org", self.preview_url._cache)
        self.preview_url._cache.pop("http://matrix.org")
        self.assertNotIn("http://matrix.org", self.preview_url._cache)

        # Check the database cache returns the correct response
        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        # Check the cache response has the same content
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )
    def test_cache_returns_correct_type(self):
        self.lookups["matrix.org"] = [(IPv4Address, "8.8.8.8")]

        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content),)
            + self.end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )

        # Check the cache returns the correct response
        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        # Check the cache response has the same content
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )

        # Clear the in-memory cache
        self.assertIn("http://matrix.org", self.preview_url._cache)
        self.preview_url._cache.pop("http://matrix.org")
        self.assertNotIn("http://matrix.org", self.preview_url._cache)

        # Check the database cache returns the correct response
        request, channel = self.make_request(
            "GET", "url_preview?url=http://matrix.org", shorthand=False
        )
        request.render(self.preview_url)
        self.pump()

        # Check the cache response has the same content
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )
Exemple #15
0
    def test_status(self):
        agent = Agent(reactor)
        response = yield agent.request("GET", b"http://localhost:9010/status")

        proto = AccumulatingProtocol()
        proto.closedDeferred = Deferred()
        response.deliverBody(proto)
        yield proto.closedDeferred

        payload = json.loads(proto.data)
        eq_(payload, {"status": "OK", "version": __version__})
Exemple #16
0
    def test_status(self):
        agent = Agent(reactor)
        response = yield agent.request("GET", b"http://localhost:9010/status")

        proto = AccumulatingProtocol()
        proto.closedDeferred = Deferred()
        response.deliverBody(proto)
        yield proto.closedDeferred

        payload = json.loads(proto.data)
        eq_(payload, {"status": "OK", "version": __version__})
def connect_logging_client(reactor, client_id):
    # This is essentially tests.server.connect_client, but disabling autoflush on
    # the client transport. This is necessary to avoid an infinite loop due to
    # sending of data via the logging transport causing additional logs to be
    # written.
    factory = reactor.tcpClients.pop(client_id)[2]
    client = factory.buildProtocol(None)
    server = AccumulatingProtocol()
    server.makeConnection(FakeTransport(client, reactor))
    client.makeConnection(FakeTransport(server, reactor, autoflush=False))

    return client, server
    def test_accept_language_config_option(self):
        """
        Accept-Language header is sent to the remote server
        """
        self.lookups["example.com"] = [(IPv4Address, "10.1.2.3")]

        # Build and make a request to the server
        channel = self.make_request(
            "GET",
            "preview_url?url=http://example.com",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        # Extract Synapse's tcp client
        client = self.reactor.tcpClients[0][2].buildProtocol(None)

        # Build a fake remote server to reply with
        server = AccumulatingProtocol()

        # Connect the two together
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))

        # Tell Synapse that it has received some data from the remote server
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content),)
            + self.end_content
        )

        # Move the reactor along until we get a response on our original channel
        self.pump()
        self.assertEqual(channel.code, 200)
        self.assertEqual(
            channel.json_body, {"og:title": "~matrix~", "og:description": "hi"}
        )

        # Check that the server received the Accept-Language header as part
        # of the request from Synapse
        self.assertIn(
            (
                b"Accept-Language: en-UK\r\n"
                b"Accept-Language: en-US;q=0.9\r\n"
                b"Accept-Language: fr;q=0.8\r\n"
                b"Accept-Language: *;q=0.7"
            ),
            server.data,
        )
    def test_oembed_rich(self):
        """Test an oEmbed endpoint which returns HTML content via the 'rich' type."""
        # Route the HTTP version to an HTTP endpoint so that the tests work.
        with patch.dict(
            "synapse.rest.media.v1.preview_url_resource._oembed_patterns",
            {
                re.compile(
                    r"http://twitter\.com/.+/status/.+"
                ): "http://publish.twitter.com/oembed",
            },
            clear=True,
        ):

            self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]

            result = {
                "version": "1.0",
                "type": "rich",
                "html": "<div>Content Preview</div>",
            }
            end_content = json.dumps(result).encode("utf-8")

            channel = self.make_request(
                "GET",
                "preview_url?url=http://twitter.com/matrixdotorg/status/12345",
                shorthand=False,
                await_result=False,
            )
            self.pump()

            client = self.reactor.tcpClients[0][2].buildProtocol(None)
            server = AccumulatingProtocol()
            server.makeConnection(FakeTransport(client, self.reactor))
            client.makeConnection(FakeTransport(server, self.reactor))
            client.dataReceived(
                (
                    b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                    b'Content-Type: application/json; charset="utf8"\r\n\r\n'
                )
                % (len(end_content),)
                + end_content
            )

            self.pump()
            self.assertEqual(channel.code, 200)
            self.assertEqual(
                channel.json_body,
                {"og:title": None, "og:description": "Content Preview"},
            )
Exemple #20
0
def connect_client(reactor: IReactorTCP, client_id: int) -> AccumulatingProtocol:
    """
    Connect a client to a fake TCP transport.

    Args:
        reactor
        factory: The connecting factory to build.
    """
    factory = reactor.tcpClients.pop(client_id)[2]
    client = factory.buildProtocol(None)
    server = AccumulatingProtocol()
    server.makeConnection(FakeTransport(client, reactor))
    client.makeConnection(FakeTransport(server, reactor))

    return client, server
Exemple #21
0
    def test_oembed_format(self):
        """Test an oEmbed endpoint which requires the format in the URL."""
        self.lookups["www.hulu.com"] = [(IPv4Address, "10.1.2.3")]

        result = {
            "version": "1.0",
            "type": "rich",
            "html": "<div>Content Preview</div>",
        }
        end_content = json.dumps(result).encode("utf-8")

        channel = self.make_request(
            "GET",
            "preview_url?url=http://www.hulu.com/watch/12345",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (
                b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                b'Content-Type: application/json; charset="utf8"\r\n\r\n'
            )
            % (len(end_content),)
            + end_content
        )

        self.pump()

        # The {format} should have been turned into json.
        self.assertIn(b"/api/oembed.json", server.data)
        # A URL parameter of format=json should be provided.
        self.assertIn(b"format=json", server.data)

        self.assertEqual(channel.code, 200)
        body = channel.json_body
        self.assertEqual(
            body,
            {
                "og:url": "http://www.hulu.com/watch/12345",
                "og:description": "Content Preview",
            },
        )
Exemple #22
0
 def setUp(self):
     super(WebSocketsProtocolWrapperTest, self).setUp()
     self.accumulatingProtocol = AccumulatingProtocol()
     self.protocol = WebSocketsProtocolWrapper(self.accumulatingProtocol)
     self.transport = StringTransportWithDisconnection()
     self.protocol.makeConnection(self.transport)
     self.transport.protocol = self.protocol
Exemple #23
0
    def test_lose_connection(self):
        """
        We log the URI correctly redacted when we lose the connection.
        """

        class HangingResource(Resource):
            """
            A Resource that strategically hangs, as if it were processing an
            answer.
            """

            def render(self, request):
                return NOT_DONE_YET

        # Set up a logging handler that we can inspect afterwards
        output = StringIO()
        handler = logging.StreamHandler(output)
        logger.addHandler(handler)
        old_level = logger.level
        logger.setLevel(10)
        self.addCleanup(logger.setLevel, old_level)
        self.addCleanup(logger.removeHandler, handler)

        # Make a resource and a Site, the resource will hang and allow us to
        # time out the request while it's 'processing'
        base_resource = Resource()
        base_resource.putChild(b'', HangingResource())
        site = SynapseSite("test", "site_tag", {}, base_resource, "1.0")

        server = site.buildProtocol(None)
        client = AccumulatingProtocol()
        client.makeConnection(FakeTransport(server, self.reactor))
        server.makeConnection(FakeTransport(client, self.reactor))

        # Send a request with an access token that will get redacted
        server.dataReceived(b"GET /?access_token=bar HTTP/1.0\r\n\r\n")
        self.pump()

        # Lose the connection
        e = Failure(Exception("Failed123"))
        server.connectionLost(e)
        handler.flush()

        # Our access token is redacted and the failure reason is logged.
        self.assertIn("/?access_token=<redacted>", output.getvalue())
        self.assertIn("Failed123", output.getvalue())
Exemple #24
0
    def test_reactor(self):
        """Apply the blacklisting reactor and ensure it properly blocks connections to particular domains and IPs."""
        agent = Agent(
            BlacklistingReactorWrapper(
                self.reactor,
                ip_whitelist=self.ip_whitelist,
                ip_blacklist=self.ip_blacklist,
            ),
        )

        # The unsafe domains and IPs should be rejected.
        for domain in (self.unsafe_domain, self.unsafe_ip):
            self.failureResultOf(
                agent.request(b"GET", b"http://" + domain), DNSLookupError
            )

        # The safe domains IPs should be accepted.
        for domain in (
            self.safe_domain,
            self.allowed_domain,
            self.safe_ip,
            self.allowed_ip,
        ):
            d = agent.request(b"GET", b"http://" + domain)

            # Grab the latest TCP connection.
            (
                host,
                port,
                client_factory,
                _timeout,
                _bindAddress,
            ) = self.reactor.tcpClients[-1]

            # Make the connection and pump data through it.
            client = client_factory.buildProtocol(None)
            server = AccumulatingProtocol()
            server.makeConnection(FakeTransport(client, self.reactor))
            client.makeConnection(FakeTransport(server, self.reactor))
            client.dataReceived(
                b"HTTP/1.0 200 OK\r\nContent-Length: 0\r\nContent-Type: text/html\r\n\r\n"
            )

            response = self.successResultOf(d)
            self.assertEqual(response.code, 200)
Exemple #25
0
    def test_oembed_rich(self):
        """Test an oEmbed endpoint which returns HTML content via the 'rich' type."""
        self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]

        result = {
            "version": "1.0",
            "type": "rich",
            # Note that this provides the author, not the title.
            "author_name": "Alice",
            "html": "<div>Content Preview</div>",
        }
        end_content = json.dumps(result).encode("utf-8")

        channel = self.make_request(
            "GET",
            "preview_url?url=http://twitter.com/matrixdotorg/status/12345",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (
                b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                b'Content-Type: application/json; charset="utf8"\r\n\r\n'
            )
            % (len(end_content),)
            + end_content
        )

        self.pump()
        self.assertEqual(channel.code, 200)
        body = channel.json_body
        self.assertEqual(
            body,
            {
                "og:url": "http://twitter.com/matrixdotorg/status/12345",
                "og:title": "Alice",
                "og:description": "Content Preview",
            },
        )
Exemple #26
0
def connect_mqtt_server(server_factory):

    server_protocol = server_factory.buildProtocol(None)
    server_transport = FakeTransport(server_protocol, True)

    client_protocol = AccumulatingProtocol()
    client_transport = FakeTransport(client_protocol, False)

    mqtt_pump = connect(server_protocol, server_transport, client_protocol,
                        client_transport, debug=False)

    return client_transport, client_protocol, mqtt_pump
Exemple #27
0
    def test_oembed_photo(self) -> None:
        """Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL."""
        self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
        self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]

        result = {
            "version": "1.0",
            "type": "photo",
            "url": "http://cdn.twitter.com/matrixdotorg",
        }
        oembed_content = json.dumps(result).encode("utf-8")

        channel = self.make_request(
            "GET",
            "preview_url?url=http://twitter.com/matrixdotorg/status/12345",
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            (b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
             b'Content-Type: application/json; charset="utf8"\r\n\r\n') %
            (len(oembed_content), ) + oembed_content)

        self.pump()

        # Ensure a second request is made to the photo URL.
        client = self.reactor.tcpClients[1][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived((b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                             b"Content-Type: image/png\r\n\r\n") %
                            (len(SMALL_PNG), ) + SMALL_PNG)

        self.pump()

        # Ensure the URL is what was requested.
        self.assertIn(b"/matrixdotorg", server.data)

        self.assertEqual(channel.code, 200)
        body = channel.json_body
        self.assertEqual(body["og:url"],
                         "http://twitter.com/matrixdotorg/status/12345")
        self._assert_small_png(body)
Exemple #28
0
    def _authorized_request(self,
                            token,
                            headers,
                            kubernetes_host=b"example.invalid."):
        """
        Get an agent using ``authenticate_with_serviceaccount`` and issue a
        request with it.

        :return bytes: The bytes of the request the agent issues.
        """
        server = AccumulatingProtocol()
        factory = Factory.forProtocol(lambda: server)
        factory.protocolConnectionMade = None

        reactor = create_reactor()
        reactor.listenTCP(80, factory)

        t = FilePath(self.useFixture(TempDir()).path)
        t = t.asBytesMode()
        serviceaccount = t.child(b"serviceaccount")
        serviceaccount.makedirs()

        serviceaccount.child(b"ca.crt").setContent(_CA_CERT_PEM)
        serviceaccount.child(b"token").setContent(token)

        environ = encode_environ({
            u"KUBERNETES_SERVICE_HOST":
            kubernetes_host.decode("ascii"),
            u"KUBERNETES_SERVICE_PORT":
            u"443"
        })
        self.patch(os, "environ", environ)

        agent = authenticate_with_serviceaccount(
            reactor,
            path=serviceaccount.asTextMode().path,
        )

        d = agent.request(b"GET", b"http://" + kubernetes_host, headers)
        assertNoResult(self, d)
        [(host, port, factory, _, _)] = reactor.tcpClients

        addr = HOST_MAP.get(kubernetes_host.decode("ascii"), None)
        self.expectThat((host, port), Equals((addr, 80)))

        pump = ConnectionCompleter(reactor).succeedOnce()
        pump.pump()

        return server.data
Exemple #29
0
    def test_blacklist_port(self) -> None:
        """Tests that blacklisting URLs with a port makes previewing such URLs
        fail with a 403 error and doesn't impact other previews.
        """
        self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]

        bad_url = quote("http://matrix.org:8888/foo")
        good_url = quote("http://matrix.org/foo")

        channel = self.make_request(
            "GET",
            "preview_url?url=" + bad_url,
            shorthand=False,
            await_result=False,
        )
        self.pump()
        self.assertEqual(channel.code, 403, channel.result)

        channel = self.make_request(
            "GET",
            "preview_url?url=" + good_url,
            shorthand=False,
            await_result=False,
        )
        self.pump()

        client = self.reactor.tcpClients[0][2].buildProtocol(None)
        server = AccumulatingProtocol()
        server.makeConnection(FakeTransport(client, self.reactor))
        client.makeConnection(FakeTransport(server, self.reactor))
        client.dataReceived(
            b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\nContent-Type: text/html\r\n\r\n"
            % (len(self.end_content), ) + self.end_content)

        self.pump()
        self.assertEqual(channel.code, 200)
Exemple #30
0
def applyPatch(patch, patchLevel="0", reactor=None):
    """
    Apply a patch to the current git repository.

    @param patch: Patch to apply
    @type patch: L{str}

    @param patchLevel: Number of directries to strip from paths in patch
    """
    proto = AccumulatingProtocol()
    done = Deferred()
    proto.closedDeferred = done
    def feedPatch(proto):
        proto.transport.write(patch)
        proto.transport.closeStdin()
    connectProtocol(
            ProcessEndpoint(reactor, "git", ("git", "apply", "--index",
                                             "-p", patchLevel)),
            proto).addCallback(feedPatch)
    def eb(_):
        # Note, we can't print why git apply failed due to https://tm.tl/#6576
        proto.closedReason.trap(ConnectionDone)
    done.addCallback(eb)
    return done
    def test_forwards_data(self):
        """
        Data received by the protocol gets passed to the wrapped
        protocol.
        """
        clock = Clock()
        wrapped_protocol = AccumulatingProtocol()
        protocol = HangCheckProtocol(wrapped_protocol, reactor=clock)
        transport = StringTransport()

        transport.protocol = protocol
        protocol.makeConnection(transport)
        protocol.dataReceived('some-data')

        self.assertEqual(wrapped_protocol.data, "some-data")
Exemple #32
0
    def _authorized_request(self, token, headers):
        """
        Get an agent using ``authenticate_with_serviceaccount`` and issue a
        request with it.

        :return bytes: The bytes of the request the agent issues.
        """
        server = AccumulatingProtocol()
        factory = Factory.forProtocol(lambda: server)
        factory.protocolConnectionMade = None

        reactor = MemoryReactor()
        reactor.listenTCP(80, factory)

        t = FilePath(self.useFixture(TempDir()).join(b""))
        serviceaccount = t.child(b"serviceaccount")
        serviceaccount.makedirs()

        serviceaccount.child(b"ca.crt").setContent(_CA_CERT_PEM)
        serviceaccount.child(b"token").setContent(token)

        self.patch(
            os,
            "environ",
            {
                b"KUBERNETES_SERVICE_HOST": b"example.invalid.",
                b"KUBERNETES_SERVICE_PORT": b"443",
            },
        )

        agent = authenticate_with_serviceaccount(
            reactor,
            path=serviceaccount.path,
        )
        agent.request(b"GET", b"http://example.invalid.", headers)

        [(host, port, factory, _, _)] = reactor.tcpClients

        self.expectThat((host, port), Equals((b"example.invalid.", 80)))

        pump = ConnectionCompleter(reactor).succeedOnce()
        pump.pump()

        return server.data
    def test_disconnect_forwarded(self):
        """
        If the connection is closed, the underlying protocol is informed.
        """
        clock = Clock()
        wrapped_protocol = AccumulatingProtocol()
        protocol = HangCheckProtocol(wrapped_protocol, reactor=clock)
        transport = StringTransport()

        transport.protocol = protocol
        protocol.makeConnection(transport)
        reason = ConnectionDone("Bye.")
        protocol.connectionLost(Failure(reason))

        self.assertTrue(wrapped_protocol.closed)
        self.assertEqual(
            wrapped_protocol.closedReason.value,
            reason,
        )
Exemple #34
0
 def lookupProtocol(names, otherRequest):
     return AccumulatingProtocol(), None
    def test_oembed_photo(self):
        """Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL."""
        # Route the HTTP version to an HTTP endpoint so that the tests work.
        with patch.dict(
            "synapse.rest.media.v1.preview_url_resource._oembed_patterns",
            {
                re.compile(
                    r"http://twitter\.com/.+/status/.+"
                ): "http://publish.twitter.com/oembed",
            },
            clear=True,
        ):

            self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
            self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]

            result = {
                "version": "1.0",
                "type": "photo",
                "url": "http://cdn.twitter.com/matrixdotorg",
            }
            oembed_content = json.dumps(result).encode("utf-8")

            end_content = (
                b"<html><head>"
                b"<title>Some Title</title>"
                b'<meta property="og:description" content="hi" />'
                b"</head></html>"
            )

            channel = self.make_request(
                "GET",
                "preview_url?url=http://twitter.com/matrixdotorg/status/12345",
                shorthand=False,
                await_result=False,
            )
            self.pump()

            client = self.reactor.tcpClients[0][2].buildProtocol(None)
            server = AccumulatingProtocol()
            server.makeConnection(FakeTransport(client, self.reactor))
            client.makeConnection(FakeTransport(server, self.reactor))
            client.dataReceived(
                (
                    b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                    b'Content-Type: application/json; charset="utf8"\r\n\r\n'
                )
                % (len(oembed_content),)
                + oembed_content
            )

            self.pump()

            client = self.reactor.tcpClients[1][2].buildProtocol(None)
            server = AccumulatingProtocol()
            server.makeConnection(FakeTransport(client, self.reactor))
            client.makeConnection(FakeTransport(server, self.reactor))
            client.dataReceived(
                (
                    b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
                    b'Content-Type: text/html; charset="utf8"\r\n\r\n'
                )
                % (len(end_content),)
                + end_content
            )

            self.pump()

            self.assertEqual(channel.code, 200)
            self.assertEqual(
                channel.json_body, {"og:title": "Some Title", "og:description": "hi"}
            )